2024-12-04 12:46:21,723 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-12-04 12:46:21,736 main DEBUG Took 0.010765 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 12:46:21,736 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 12:46:21,737 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 12:46:21,738 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 12:46:21,739 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 12:46:21,752 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 12:46:21,767 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 12:46:21,769 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 12:46:21,770 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 12:46:21,770 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 12:46:21,771 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 12:46:21,771 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 12:46:21,772 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 12:46:21,773 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 12:46:21,774 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 12:46:21,774 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 12:46:21,775 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 12:46:21,775 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 12:46:21,776 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 12:46:21,776 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 12:46:21,777 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 12:46:21,777 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 12:46:21,778 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 12:46:21,778 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 12:46:21,778 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 12:46:21,779 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 12:46:21,779 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 12:46:21,780 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 12:46:21,780 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 12:46:21,781 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 12:46:21,781 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 12:46:21,781 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 12:46:21,783 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 12:46:21,785 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 12:46:21,787 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 12:46:21,788 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 12:46:21,790 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 12:46:21,790 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 12:46:21,800 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 12:46:21,803 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 12:46:21,805 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 12:46:21,806 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 12:46:21,806 main DEBUG createAppenders(={Console}) 2024-12-04 12:46:21,807 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 initialized 2024-12-04 12:46:21,807 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-12-04 12:46:21,808 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 OK. 2024-12-04 12:46:21,808 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 12:46:21,809 main DEBUG OutputStream closed 2024-12-04 12:46:21,809 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 12:46:21,810 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 12:46:21,810 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3c73951 OK 2024-12-04 12:46:21,887 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 12:46:21,889 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 12:46:21,890 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 12:46:21,891 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 12:46:21,892 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 12:46:21,892 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 12:46:21,892 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 12:46:21,893 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 12:46:21,893 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 12:46:21,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 12:46:21,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 12:46:21,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 12:46:21,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 12:46:21,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 12:46:21,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 12:46:21,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 12:46:21,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 12:46:21,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 12:46:21,899 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 12:46:21,899 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@61fe30) with optional ClassLoader: null 2024-12-04 12:46:21,899 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 12:46:21,900 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@61fe30] started OK. 2024-12-04T12:46:21,934 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackup timeout: 13 mins 2024-12-04 12:46:21,937 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 12:46:21,937 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T12:46:22,273 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354 2024-12-04T12:46:22,279 DEBUG [Time-limited test {}] impl.BackupManager(124): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-12-04T12:46:22,281 DEBUG [Time-limited test {}] impl.BackupManager(154): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:46:22,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T12:46:22,301 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/cluster_4dfbfc9e-7777-d399-4020-07547724c305, deleteOnExit=true 2024-12-04T12:46:22,302 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T12:46:22,303 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/test.cache.data in system properties and HBase conf 2024-12-04T12:46:22,304 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T12:46:22,304 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir in system properties and HBase conf 2024-12-04T12:46:22,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T12:46:22,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T12:46:22,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T12:46:22,406 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T12:46:22,506 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T12:46:22,511 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T12:46:22,511 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T12:46:22,512 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T12:46:22,512 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T12:46:22,513 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T12:46:22,513 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T12:46:22,514 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T12:46:22,514 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T12:46:22,515 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T12:46:22,515 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/nfs.dump.dir in system properties and HBase conf 2024-12-04T12:46:22,516 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/java.io.tmpdir in system properties and HBase conf 2024-12-04T12:46:22,516 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T12:46:22,517 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T12:46:22,517 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T12:46:23,602 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T12:46:23,675 INFO [Time-limited test {}] log.Log(170): Logging initialized @2692ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T12:46:23,752 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T12:46:23,814 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T12:46:23,835 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T12:46:23,835 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T12:46:23,837 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T12:46:23,850 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T12:46:23,853 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@205baae5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,AVAILABLE} 2024-12-04T12:46:23,854 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@325f8c88{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T12:46:24,021 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1d57f2ce{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/java.io.tmpdir/jetty-localhost-38955-hadoop-hdfs-3_4_1-tests_jar-_-any-2305358741687715978/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T12:46:24,030 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@382e7786{HTTP/1.1, (http/1.1)}{localhost:38955} 2024-12-04T12:46:24,031 INFO [Time-limited test {}] server.Server(415): Started @3049ms 2024-12-04T12:46:24,479 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T12:46:24,486 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T12:46:24,487 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T12:46:24,487 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T12:46:24,487 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T12:46:24,488 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@347c3eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,AVAILABLE} 2024-12-04T12:46:24,488 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@308dcebb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T12:46:24,583 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d3c17d4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/java.io.tmpdir/jetty-localhost-36089-hadoop-hdfs-3_4_1-tests_jar-_-any-16429264424230820472/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T12:46:24,583 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@575d15e0{HTTP/1.1, (http/1.1)}{localhost:36089} 2024-12-04T12:46:24,584 INFO [Time-limited test {}] server.Server(415): Started @3603ms 2024-12-04T12:46:24,642 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T12:46:25,456 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/cluster_4dfbfc9e-7777-d399-4020-07547724c305/data/data1/current/BP-117991357-172.17.0.3-1733316383096/current, will proceed with Du for space computation calculation, 2024-12-04T12:46:25,456 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/cluster_4dfbfc9e-7777-d399-4020-07547724c305/data/data2/current/BP-117991357-172.17.0.3-1733316383096/current, will proceed with Du for space computation calculation, 2024-12-04T12:46:25,488 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T12:46:25,533 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1bf4de17a445b631 with lease ID 0xce40fb89545184ea: Processing first storage report for DS-fb7dda59-85b1-4c17-b284-5c4de7371519 from datanode DatanodeRegistration(127.0.0.1:44611, datanodeUuid=74cbd0c3-db18-4c04-abbd-e9560e582016, infoPort=45053, infoSecurePort=0, ipcPort=37309, storageInfo=lv=-57;cid=testClusterID;nsid=1388320770;c=1733316383096) 2024-12-04T12:46:25,534 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1bf4de17a445b631 with lease ID 0xce40fb89545184ea: from storage DS-fb7dda59-85b1-4c17-b284-5c4de7371519 node DatanodeRegistration(127.0.0.1:44611, datanodeUuid=74cbd0c3-db18-4c04-abbd-e9560e582016, infoPort=45053, infoSecurePort=0, ipcPort=37309, storageInfo=lv=-57;cid=testClusterID;nsid=1388320770;c=1733316383096), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T12:46:25,534 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1bf4de17a445b631 with lease ID 0xce40fb89545184ea: Processing first storage report for DS-d90f3c63-e98e-4546-89fc-ed8b8580234a from datanode DatanodeRegistration(127.0.0.1:44611, datanodeUuid=74cbd0c3-db18-4c04-abbd-e9560e582016, infoPort=45053, infoSecurePort=0, ipcPort=37309, storageInfo=lv=-57;cid=testClusterID;nsid=1388320770;c=1733316383096) 2024-12-04T12:46:25,534 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1bf4de17a445b631 with lease ID 0xce40fb89545184ea: from storage DS-d90f3c63-e98e-4546-89fc-ed8b8580234a node DatanodeRegistration(127.0.0.1:44611, datanodeUuid=74cbd0c3-db18-4c04-abbd-e9560e582016, infoPort=45053, infoSecurePort=0, ipcPort=37309, storageInfo=lv=-57;cid=testClusterID;nsid=1388320770;c=1733316383096), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T12:46:25,584 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354 2024-12-04T12:46:25,677 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/cluster_4dfbfc9e-7777-d399-4020-07547724c305/zookeeper_0, clientPort=60209, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/cluster_4dfbfc9e-7777-d399-4020-07547724c305/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/cluster_4dfbfc9e-7777-d399-4020-07547724c305/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T12:46:25,689 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60209 2024-12-04T12:46:25,703 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T12:46:25,708 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T12:46:25,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741825_1001 (size=7) 2024-12-04T12:46:26,319 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a with version=8 2024-12-04T12:46:26,319 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging 2024-12-04T12:46:26,331 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(136): Setting Master Port to random. 2024-12-04T12:46:26,331 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(143): Setting RegionServer Port to random. 2024-12-04T12:46:26,331 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(153): Setting RS InfoServer Port to random. 2024-12-04T12:46:26,332 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(161): Setting Master InfoServer Port to random. 2024-12-04T12:46:26,409 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T12:46:26,626 INFO [Time-limited test {}] client.ConnectionUtils(128): master/938b79ed23df:0 server-side Connection retries=45 2024-12-04T12:46:26,635 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T12:46:26,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T12:46:26,640 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T12:46:26,640 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T12:46:26,640 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T12:46:26,771 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T12:46:26,834 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T12:46:26,843 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T12:46:26,847 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T12:46:26,872 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 147840 (auto-detected) 2024-12-04T12:46:26,873 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-04T12:46:26,894 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36049 2024-12-04T12:46:26,919 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36049 connecting to ZooKeeper ensemble=127.0.0.1:60209 2024-12-04T12:46:27,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:360490x0, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T12:46:27,028 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36049-0x101a18ee11e0000 connected 2024-12-04T12:46:27,154 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T12:46:27,156 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T12:46:27,164 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-04T12:46:27,182 INFO [Time-limited test {}] log.Log(170): Logging initialized @6201ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T12:46:27,219 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-12-04T12:46:27,220 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-12-04T12:46:27,221 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-12-04T12:46:27,221 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-12-04T12:46:27,222 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-12-04T12:46:27,223 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2024-12-04T12:46:27,223 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-12-04T12:46:27,224 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-12-04T12:46:27,225 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-12-04T12:46:27,281 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 42697 2024-12-04T12:46:27,282 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-12-04T12:46:27,294 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-04T12:46:27,296 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3763b03b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,AVAILABLE} 2024-12-04T12:46:27,296 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-04T12:46:27,297 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@390508af{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-12-04T12:46:27,334 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-12-04T12:46:27,340 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T12:46:27,340 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T12:46:27,341 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T12:46:27,345 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-04T12:46:27,874 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@3bb1047d{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-12-04T12:46:27,879 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@678f6157{HTTP/1.1, (http/1.1)}{0.0.0.0:42697} 2024-12-04T12:46:27,879 INFO [Time-limited test {}] server.Server(415): Started @6898ms 2024-12-04T12:46:27,882 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a, hbase.cluster.distributed=false 2024-12-04T12:46:27,904 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-12-04T12:46:27,908 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36049 2024-12-04T12:46:27,908 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36049 2024-12-04T12:46:27,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36049 2024-12-04T12:46:27,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36049 2024-12-04T12:46:27,910 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36049 2024-12-04T12:46:28,023 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/938b79ed23df:0 server-side Connection retries=45 2024-12-04T12:46:28,024 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T12:46:28,024 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T12:46:28,025 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T12:46:28,025 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T12:46:28,025 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T12:46:28,028 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T12:46:28,030 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T12:46:28,031 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38879 2024-12-04T12:46:28,032 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38879 connecting to ZooKeeper ensemble=127.0.0.1:60209 2024-12-04T12:46:28,033 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T12:46:28,038 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T12:46:28,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388790x0, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T12:46:28,111 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:388790x0, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-04T12:46:28,111 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38879-0x101a18ee11e0001 connected 2024-12-04T12:46:28,113 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-12-04T12:46:28,113 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-12-04T12:46:28,113 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-12-04T12:46:28,113 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-12-04T12:46:28,113 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-12-04T12:46:28,114 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2024-12-04T12:46:28,114 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-12-04T12:46:28,114 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-12-04T12:46:28,115 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-12-04T12:46:28,116 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 38869 2024-12-04T12:46:28,116 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-12-04T12:46:28,118 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-04T12:46:28,118 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@749d911f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,AVAILABLE} 2024-12-04T12:46:28,118 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-04T12:46:28,119 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@54a4c9e0{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-12-04T12:46:28,134 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-12-04T12:46:28,135 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T12:46:28,135 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T12:46:28,136 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T12:46:28,137 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-04T12:46:28,140 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@49ec94de{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-12-04T12:46:28,141 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13520b09{HTTP/1.1, (http/1.1)}{0.0.0.0:38869} 2024-12-04T12:46:28,141 INFO [Time-limited test {}] server.Server(415): Started @7160ms 2024-12-04T12:46:28,145 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T12:46:28,153 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T12:46:28,156 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2024-12-04T12:46:28,161 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-12-04T12:46:28,162 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38879 2024-12-04T12:46:28,162 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38879 2024-12-04T12:46:28,163 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38879 2024-12-04T12:46:28,163 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38879 2024-12-04T12:46:28,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38879 2024-12-04T12:46:28,178 DEBUG [M:0;938b79ed23df:36049 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;938b79ed23df:36049 2024-12-04T12:46:28,185 INFO [master/938b79ed23df:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-12-04T12:46:28,187 INFO [master/938b79ed23df:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@6df710df{HTTP/1.1, (http/1.1)}{0.0.0.0:42993} 2024-12-04T12:46:28,188 INFO [master/938b79ed23df:0:becomeActiveMaster {}] server.Server(415): Started @7207ms 2024-12-04T12:46:28,188 INFO [master/938b79ed23df:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /1/backup-masters/938b79ed23df,36049,1733316386472 2024-12-04T12:46:28,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-04T12:46:28,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-04T12:46:28,204 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/938b79ed23df,36049,1733316386472 2024-12-04T12:46:28,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2024-12-04T12:46:28,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:46:28,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:46:28,228 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on existing znode=/1/master 2024-12-04T12:46:28,229 INFO [master/938b79ed23df:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/938b79ed23df,36049,1733316386472 from backup master directory 2024-12-04T12:46:28,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-04T12:46:28,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/938b79ed23df,36049,1733316386472 2024-12-04T12:46:28,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-04T12:46:28,236 WARN [master/938b79ed23df:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T12:46:28,237 INFO [master/938b79ed23df:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=938b79ed23df,36049,1733316386472 2024-12-04T12:46:28,239 INFO [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T12:46:28,241 INFO [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T12:46:28,309 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase.id] with ID: 6f9b823b-072d-46db-86e7-8896bdc5736e 2024-12-04T12:46:28,309 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.tmp/hbase.id 2024-12-04T12:46:28,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741826_1002 (size=42) 2024-12-04T12:46:28,722 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.tmp/hbase.id]:[hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase.id] 2024-12-04T12:46:28,770 INFO [master/938b79ed23df:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T12:46:28,774 INFO [master/938b79ed23df:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T12:46:28,792 INFO [master/938b79ed23df:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-04T12:46:28,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:46:28,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:46:28,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741827_1003 (size=196) 2024-12-04T12:46:29,255 INFO [master/938b79ed23df:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:46:29,257 INFO [master/938b79ed23df:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T12:46:29,262 INFO [master/938b79ed23df:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-12-04T12:46:29,264 INFO [master/938b79ed23df:0:becomeActiveMaster {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-12-04T12:46:29,273 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.doInit(RegionGroupingProvider.java:156) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.init(AbstractWALProvider.java:126) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:263) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T12:46:29,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741828_1004 (size=1189) 2024-12-04T12:46:29,719 INFO [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store 2024-12-04T12:46:29,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741829_1005 (size=34) 2024-12-04T12:46:30,141 INFO [master/938b79ed23df:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T12:46:30,145 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:30,146 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T12:46:30,146 INFO [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T12:46:30,147 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T12:46:30,148 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T12:46:30,148 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T12:46:30,149 INFO [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T12:46:30,150 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733316390146Disabling compacts and flushes for region at 1733316390146Disabling writes for close at 1733316390148 (+2 ms)Writing region close event to WAL at 1733316390149 (+1 ms)Closed at 1733316390149 2024-12-04T12:46:30,152 WARN [master/938b79ed23df:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/.initializing 2024-12-04T12:46:30,152 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/WALs/938b79ed23df,36049,1733316386472 2024-12-04T12:46:30,157 INFO [master/938b79ed23df:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T12:46:30,162 INFO [master/938b79ed23df:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 938b79ed23df,36049,1733316386472.regiongroup-0 2024-12-04T12:46:30,177 INFO [master/938b79ed23df:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=938b79ed23df%2C36049%2C1733316386472.938b79ed23df%2C36049%2C1733316386472.regiongroup-0, suffix=, logDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/WALs/938b79ed23df,36049,1733316386472, archiveDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/oldWALs, maxLogs=10 2024-12-04T12:46:30,207 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/WALs/938b79ed23df,36049,1733316386472/938b79ed23df%2C36049%2C1733316386472.938b79ed23df%2C36049%2C1733316386472.regiongroup-0.1733316390182, exclude list is [], retry=0 2024-12-04T12:46:30,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:46:30,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-04T12:46:30,257 INFO [master/938b79ed23df:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/WALs/938b79ed23df,36049,1733316386472/938b79ed23df%2C36049%2C1733316386472.938b79ed23df%2C36049%2C1733316386472.regiongroup-0.1733316390182 2024-12-04T12:46:30,258 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:46:30,258 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T12:46:30,259 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:30,261 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T12:46:30,262 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T12:46:30,299 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T12:46:30,324 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T12:46:30,327 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:30,329 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T12:46:30,330 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T12:46:30,334 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T12:46:30,334 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:30,335 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:46:30,336 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T12:46:30,339 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T12:46:30,339 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:30,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:46:30,342 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T12:46:30,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T12:46:30,345 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:30,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:46:30,347 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T12:46:30,350 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T12:46:30,352 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T12:46:30,358 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T12:46:30,358 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T12:46:30,362 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T12:46:30,365 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T12:46:30,371 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T12:46:30,373 INFO [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11332386080, jitterRate=0.05541069805622101}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T12:46:30,378 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733316390276Initializing all the Stores at 1733316390279 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733316390279Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316390281 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316390281Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316390281Cleaning up temporary data from old regions at 1733316390358 (+77 ms)Region opened successfully at 1733316390378 (+20 ms) 2024-12-04T12:46:30,379 INFO [master/938b79ed23df:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T12:46:30,408 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4db3e585, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=938b79ed23df/172.17.0.3:0 2024-12-04T12:46:30,436 INFO [master/938b79ed23df:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T12:46:30,446 INFO [master/938b79ed23df:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T12:46:30,446 INFO [master/938b79ed23df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T12:46:30,449 INFO [master/938b79ed23df:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T12:46:30,451 INFO [master/938b79ed23df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T12:46:30,457 INFO [master/938b79ed23df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-04T12:46:30,458 INFO [master/938b79ed23df:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T12:46:30,481 INFO [master/938b79ed23df:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T12:46:30,489 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2024-12-04T12:46:30,552 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/balancer already deleted, retry=false 2024-12-04T12:46:30,555 INFO [master/938b79ed23df:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T12:46:30,556 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2024-12-04T12:46:30,560 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/normalizer already deleted, retry=false 2024-12-04T12:46:30,562 INFO [master/938b79ed23df:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T12:46:30,566 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2024-12-04T12:46:30,576 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/split already deleted, retry=false 2024-12-04T12:46:30,578 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2024-12-04T12:46:30,585 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/merge already deleted, retry=false 2024-12-04T12:46:30,601 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T12:46:30,610 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/snapshot-cleanup already deleted, retry=false 2024-12-04T12:46:30,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-12-04T12:46:30,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-12-04T12:46:30,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:46:30,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:46:30,635 INFO [master/938b79ed23df:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=938b79ed23df,36049,1733316386472, sessionid=0x101a18ee11e0000, setting cluster-up flag (Was=false) 2024-12-04T12:46:30,642 INFO [master/938b79ed23df:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2024-12-04T12:46:30,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:46:30,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:46:30,685 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2024-12-04T12:46:30,687 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=938b79ed23df,36049,1733316386472 2024-12-04T12:46:30,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:46:30,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:46:30,727 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2024-12-04T12:46:30,728 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=938b79ed23df,36049,1733316386472 2024-12-04T12:46:30,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:46:30,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:46:30,768 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2024-12-04T12:46:30,770 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=938b79ed23df,36049,1733316386472 2024-12-04T12:46:30,775 INFO [master/938b79ed23df:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T12:46:30,844 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T12:46:30,852 INFO [master/938b79ed23df:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T12:46:30,858 INFO [master/938b79ed23df:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T12:46:30,863 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 938b79ed23df,36049,1733316386472 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T12:46:30,870 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/938b79ed23df:0, corePoolSize=5, maxPoolSize=5 2024-12-04T12:46:30,870 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/938b79ed23df:0, corePoolSize=5, maxPoolSize=5 2024-12-04T12:46:30,870 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/938b79ed23df:0, corePoolSize=5, maxPoolSize=5 2024-12-04T12:46:30,870 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/938b79ed23df:0, corePoolSize=5, maxPoolSize=5 2024-12-04T12:46:30,870 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(746): ClusterId : 6f9b823b-072d-46db-86e7-8896bdc5736e 2024-12-04T12:46:30,870 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/938b79ed23df:0, corePoolSize=10, maxPoolSize=10 2024-12-04T12:46:30,870 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/938b79ed23df:0, corePoolSize=1, maxPoolSize=1 2024-12-04T12:46:30,871 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/938b79ed23df:0, corePoolSize=2, maxPoolSize=2 2024-12-04T12:46:30,871 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/938b79ed23df:0, corePoolSize=1, maxPoolSize=1 2024-12-04T12:46:30,872 INFO [master/938b79ed23df:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733316420872 2024-12-04T12:46:30,872 INFO [RS:0;938b79ed23df:38879 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2024-12-04T12:46:30,873 INFO [master/938b79ed23df:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T12:46:30,873 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T12:46:30,874 INFO [master/938b79ed23df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T12:46:30,876 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T12:46:30,877 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T12:46:30,877 INFO [master/938b79ed23df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T12:46:30,878 INFO [master/938b79ed23df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2024-12-04T12:46:30,878 INFO [master/938b79ed23df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T12:46:30,878 INFO [master/938b79ed23df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T12:46:30,878 INFO [master/938b79ed23df:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T12:46:30,879 INFO [master/938b79ed23df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:30,882 INFO [master/938b79ed23df:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T12:46:30,883 INFO [master/938b79ed23df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T12:46:30,883 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:30,884 INFO [master/938b79ed23df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T12:46:30,884 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T12:46:30,887 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T12:46:30,887 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2024-12-04T12:46:30,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741831_1007 (size=1321) 2024-12-04T12:46:30,895 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2024-12-04T12:46:30,895 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T12:46:30,895 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f767245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:30,895 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:46:30,903 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T12:46:30,904 DEBUG [RS:0;938b79ed23df:38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cda856, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=938b79ed23df/172.17.0.3:0 2024-12-04T12:46:30,906 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:46:30,923 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;938b79ed23df:38879 2024-12-04T12:46:30,926 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:46:30,927 INFO [RS:0;938b79ed23df:38879 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T12:46:30,927 INFO [RS:0;938b79ed23df:38879 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T12:46:30,927 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T12:46:30,930 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(2659): reportForDuty to master=938b79ed23df,36049,1733316386472 with port=38879, startcode=1733316387992 2024-12-04T12:46:30,933 DEBUG [RS:0;938b79ed23df:38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T12:46:30,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:46:30,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:46:30,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce6dbee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:30,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:46:30,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:46:30,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:46:30,951 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:46:30,951 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47243, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T12:46:30,957 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36049 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-04T12:46:30,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e4ef5a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:30,959 INFO [master/938b79ed23df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-12-04T12:46:30,961 INFO [master/938b79ed23df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T12:46:30,961 INFO [master/938b79ed23df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T12:46:30,963 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/938b79ed23df:0:becomeActiveMaster-HFileCleaner.large.0-1733316390962,5,FailOnTimeoutGroup] 2024-12-04T12:46:30,963 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/938b79ed23df:0:becomeActiveMaster-HFileCleaner.small.0-1733316390963,5,FailOnTimeoutGroup] 2024-12-04T12:46:30,963 INFO [master/938b79ed23df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:30,964 INFO [master/938b79ed23df:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T12:46:30,965 INFO [master/938b79ed23df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:30,965 INFO [master/938b79ed23df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:30,976 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-04T12:46:30,976 WARN [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-04T12:46:31,078 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(2659): reportForDuty to master=938b79ed23df,36049,1733316386472 with port=38879, startcode=1733316387992 2024-12-04T12:46:31,081 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36049 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 938b79ed23df,38879,1733316387992 2024-12-04T12:46:31,083 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36049 {}] master.ServerManager(517): Registering regionserver=938b79ed23df,38879,1733316387992 2024-12-04T12:46:31,093 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:46:31,093 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41555 2024-12-04T12:46:31,093 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=42697 2024-12-04T12:46:31,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-12-04T12:46:31,123 DEBUG [RS:0;938b79ed23df:38879 {}] zookeeper.ZKUtil(111): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on existing znode=/1/rs/938b79ed23df,38879,1733316387992 2024-12-04T12:46:31,124 WARN [RS:0;938b79ed23df:38879 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T12:46:31,124 INFO [RS:0;938b79ed23df:38879 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-12-04T12:46:31,124 INFO [RS:0;938b79ed23df:38879 {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-12-04T12:46:31,124 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992 2024-12-04T12:46:31,126 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [938b79ed23df,38879,1733316387992] 2024-12-04T12:46:31,147 INFO [RS:0;938b79ed23df:38879 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T12:46:31,158 INFO [RS:0;938b79ed23df:38879 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T12:46:31,163 INFO [RS:0;938b79ed23df:38879 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T12:46:31,163 INFO [RS:0;938b79ed23df:38879 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:31,164 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T12:46:31,169 INFO [RS:0;938b79ed23df:38879 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T12:46:31,170 INFO [RS:0;938b79ed23df:38879 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:31,171 DEBUG [RS:0;938b79ed23df:38879 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/938b79ed23df:0, corePoolSize=1, maxPoolSize=1 2024-12-04T12:46:31,171 DEBUG [RS:0;938b79ed23df:38879 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/938b79ed23df:0, corePoolSize=1, maxPoolSize=1 2024-12-04T12:46:31,171 DEBUG [RS:0;938b79ed23df:38879 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/938b79ed23df:0, corePoolSize=1, maxPoolSize=1 2024-12-04T12:46:31,171 DEBUG [RS:0;938b79ed23df:38879 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/938b79ed23df:0, corePoolSize=1, maxPoolSize=1 2024-12-04T12:46:31,171 DEBUG [RS:0;938b79ed23df:38879 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/938b79ed23df:0, corePoolSize=1, maxPoolSize=1 2024-12-04T12:46:31,171 DEBUG [RS:0;938b79ed23df:38879 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/938b79ed23df:0, corePoolSize=2, maxPoolSize=2 2024-12-04T12:46:31,171 DEBUG [RS:0;938b79ed23df:38879 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/938b79ed23df:0, corePoolSize=1, maxPoolSize=1 2024-12-04T12:46:31,171 DEBUG [RS:0;938b79ed23df:38879 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/938b79ed23df:0, corePoolSize=1, maxPoolSize=1 2024-12-04T12:46:31,172 DEBUG [RS:0;938b79ed23df:38879 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/938b79ed23df:0, corePoolSize=1, maxPoolSize=1 2024-12-04T12:46:31,172 DEBUG [RS:0;938b79ed23df:38879 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/938b79ed23df:0, corePoolSize=1, maxPoolSize=1 2024-12-04T12:46:31,172 DEBUG [RS:0;938b79ed23df:38879 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/938b79ed23df:0, corePoolSize=1, maxPoolSize=1 2024-12-04T12:46:31,172 DEBUG [RS:0;938b79ed23df:38879 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/938b79ed23df:0, corePoolSize=1, maxPoolSize=1 2024-12-04T12:46:31,172 DEBUG [RS:0;938b79ed23df:38879 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0, corePoolSize=3, maxPoolSize=3 2024-12-04T12:46:31,172 DEBUG [RS:0;938b79ed23df:38879 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/938b79ed23df:0, corePoolSize=3, maxPoolSize=3 2024-12-04T12:46:31,173 INFO [RS:0;938b79ed23df:38879 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:31,173 INFO [RS:0;938b79ed23df:38879 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:31,173 INFO [RS:0;938b79ed23df:38879 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:31,173 INFO [RS:0;938b79ed23df:38879 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:31,173 INFO [RS:0;938b79ed23df:38879 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:31,173 INFO [RS:0;938b79ed23df:38879 {}] hbase.ChoreService(168): Chore ScheduledChore name=938b79ed23df,38879,1733316387992-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T12:46:31,189 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T12:46:31,191 INFO [RS:0;938b79ed23df:38879 {}] hbase.ChoreService(168): Chore ScheduledChore name=938b79ed23df,38879,1733316387992-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:31,191 INFO [RS:0;938b79ed23df:38879 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:31,191 INFO [RS:0;938b79ed23df:38879 {}] regionserver.Replication(171): 938b79ed23df,38879,1733316387992 started 2024-12-04T12:46:31,206 INFO [RS:0;938b79ed23df:38879 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:31,207 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1482): Serving as 938b79ed23df,38879,1733316387992, RpcServer on 938b79ed23df/172.17.0.3:38879, sessionid=0x101a18ee11e0001 2024-12-04T12:46:31,207 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T12:46:31,207 DEBUG [RS:0;938b79ed23df:38879 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 938b79ed23df,38879,1733316387992 2024-12-04T12:46:31,208 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '938b79ed23df,38879,1733316387992' 2024-12-04T12:46:31,208 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2024-12-04T12:46:31,209 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2024-12-04T12:46:31,209 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T12:46:31,209 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2024-12-04T12:46:31,210 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '938b79ed23df,38879,1733316387992' 2024-12-04T12:46:31,210 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T12:46:31,210 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T12:46:31,211 INFO [RS:0;938b79ed23df:38879 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2024-12-04T12:46:31,211 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2024-12-04T12:46:31,211 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T12:46:31,211 DEBUG [RS:0;938b79ed23df:38879 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 938b79ed23df,38879,1733316387992 2024-12-04T12:46:31,211 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '938b79ed23df,38879,1733316387992' 2024-12-04T12:46:31,211 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2024-12-04T12:46:31,212 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2024-12-04T12:46:31,212 DEBUG [RS:0;938b79ed23df:38879 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T12:46:31,212 INFO [RS:0;938b79ed23df:38879 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T12:46:31,212 INFO [RS:0;938b79ed23df:38879 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T12:46:31,295 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T12:46:31,296 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:46:31,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741832_1008 (size=32) 2024-12-04T12:46:31,316 INFO [RS:0;938b79ed23df:38879 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T12:46:31,317 INFO [RS:0;938b79ed23df:38879 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 938b79ed23df,38879,1733316387992.regiongroup-0 2024-12-04T12:46:31,320 INFO [RS:0;938b79ed23df:38879 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0, suffix=, logDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992, archiveDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs, maxLogs=32 2024-12-04T12:46:31,333 DEBUG [RS:0;938b79ed23df:38879 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316391322, exclude list is [], retry=0 2024-12-04T12:46:31,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:46:31,342 INFO [RS:0;938b79ed23df:38879 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316391322 2024-12-04T12:46:31,342 DEBUG [RS:0;938b79ed23df:38879 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:46:31,707 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:31,710 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T12:46:31,712 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T12:46:31,712 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:31,713 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T12:46:31,713 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T12:46:31,715 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T12:46:31,716 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:31,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T12:46:31,717 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T12:46:31,719 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T12:46:31,719 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:31,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T12:46:31,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T12:46:31,722 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T12:46:31,722 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:31,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T12:46:31,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T12:46:31,725 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740 2024-12-04T12:46:31,726 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740 2024-12-04T12:46:31,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T12:46:31,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T12:46:31,730 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T12:46:31,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T12:46:31,736 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T12:46:31,737 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10291000640, jitterRate=-0.041575878858566284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T12:46:31,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733316391707Initializing all the Stores at 1733316391709 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733316391709Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733316391709Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316391709Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733316391709Cleaning up temporary data from old regions at 1733316391729 (+20 ms)Region opened successfully at 1733316391739 (+10 ms) 2024-12-04T12:46:31,740 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T12:46:31,740 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T12:46:31,740 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T12:46:31,740 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T12:46:31,740 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T12:46:31,741 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T12:46:31,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733316391740Disabling compacts and flushes for region at 1733316391740Disabling writes for close at 1733316391740Writing region close event to WAL at 1733316391741 (+1 ms)Closed at 1733316391741 2024-12-04T12:46:31,746 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T12:46:31,746 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T12:46:31,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T12:46:31,762 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T12:46:31,764 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T12:46:31,917 DEBUG [938b79ed23df:36049 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T12:46:31,928 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:31,934 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 938b79ed23df,38879,1733316387992, state=OPENING 2024-12-04T12:46:31,977 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T12:46:31,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:46:31,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:46:31,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-04T12:46:31,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-04T12:46:31,987 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T12:46:31,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:46:32,160 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T12:46:32,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47471, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T12:46:32,171 INFO [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T12:46:32,172 INFO [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-12-04T12:46:32,172 INFO [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-12-04T12:46:32,172 INFO [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T12:46:32,173 INFO [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T12:46:32,176 INFO [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=938b79ed23df%2C38879%2C1733316387992.meta, suffix=.meta, logDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992, archiveDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs, maxLogs=32 2024-12-04T12:46:32,189 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316392177.meta, exclude list is [], retry=0 2024-12-04T12:46:32,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:46:32,196 INFO [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316392177.meta 2024-12-04T12:46:32,197 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:46:32,197 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T12:46:32,203 INFO [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:46:32,204 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T12:46:32,207 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T12:46:32,207 INFO [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T12:46:32,211 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T12:46:32,212 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:32,212 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T12:46:32,212 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T12:46:32,216 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T12:46:32,217 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T12:46:32,217 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:32,218 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T12:46:32,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T12:46:32,220 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T12:46:32,221 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:32,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T12:46:32,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T12:46:32,223 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T12:46:32,224 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:32,224 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T12:46:32,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T12:46:32,226 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T12:46:32,226 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:32,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T12:46:32,228 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T12:46:32,229 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740 2024-12-04T12:46:32,232 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740 2024-12-04T12:46:32,234 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T12:46:32,234 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T12:46:32,235 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T12:46:32,237 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T12:46:32,239 INFO [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9580093600, jitterRate=-0.10778425633907318}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T12:46:32,239 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T12:46:32,240 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733316392213Writing region info on filesystem at 1733316392213Initializing all the Stores at 1733316392215 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733316392215Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733316392215Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316392215Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733316392216 (+1 ms)Cleaning up temporary data from old regions at 1733316392234 (+18 ms)Running coprocessor post-open hooks at 1733316392239 (+5 ms)Region opened successfully at 1733316392240 (+1 ms) 2024-12-04T12:46:32,246 INFO [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733316392152 2024-12-04T12:46:32,256 DEBUG [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T12:46:32,256 INFO [RS_OPEN_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T12:46:32,258 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:32,260 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 938b79ed23df,38879,1733316387992, state=OPEN 2024-12-04T12:46:32,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-12-04T12:46:32,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-12-04T12:46:32,269 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-04T12:46:32,269 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-04T12:46:32,269 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=938b79ed23df,38879,1733316387992 2024-12-04T12:46:32,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T12:46:32,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=938b79ed23df,38879,1733316387992 in 280 msec 2024-12-04T12:46:32,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T12:46:32,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 523 msec 2024-12-04T12:46:32,284 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T12:46:32,284 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T12:46:32,304 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:46:32,305 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:46:32,325 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:46:32,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54337, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:46:32,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5470 sec 2024-12-04T12:46:32,353 INFO [master/938b79ed23df:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733316392353, completionTime=-1 2024-12-04T12:46:32,355 INFO [master/938b79ed23df:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T12:46:32,356 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T12:46:32,379 INFO [master/938b79ed23df:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T12:46:32,380 INFO [master/938b79ed23df:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733316452380 2024-12-04T12:46:32,380 INFO [master/938b79ed23df:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733316512380 2024-12-04T12:46:32,380 INFO [master/938b79ed23df:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-12-04T12:46:32,382 INFO [master/938b79ed23df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=938b79ed23df,36049,1733316386472-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:32,382 INFO [master/938b79ed23df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=938b79ed23df,36049,1733316386472-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:32,382 INFO [master/938b79ed23df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=938b79ed23df,36049,1733316386472-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:32,384 INFO [master/938b79ed23df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-938b79ed23df:36049, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:32,384 INFO [master/938b79ed23df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:32,384 INFO [master/938b79ed23df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:32,390 DEBUG [master/938b79ed23df:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T12:46:32,414 INFO [master/938b79ed23df:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.177sec 2024-12-04T12:46:32,415 INFO [master/938b79ed23df:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T12:46:32,416 INFO [master/938b79ed23df:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T12:46:32,417 INFO [master/938b79ed23df:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T12:46:32,418 INFO [master/938b79ed23df:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T12:46:32,418 INFO [master/938b79ed23df:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T12:46:32,419 INFO [master/938b79ed23df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=938b79ed23df,36049,1733316386472-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T12:46:32,419 INFO [master/938b79ed23df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=938b79ed23df,36049,1733316386472-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T12:46:32,427 DEBUG [master/938b79ed23df:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T12:46:32,427 INFO [master/938b79ed23df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T12:46:32,428 INFO [master/938b79ed23df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=938b79ed23df,36049,1733316386472-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T12:46:32,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@118d4c44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:32,474 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T12:46:32,475 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T12:46:32,477 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:46:32,477 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:46:32,483 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:46:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:46:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:46:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@696e9674, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:46:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:46:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:46:32,488 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:46:32,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ee819a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:32,490 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:46:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:46:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:46:32,500 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:46:32,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=938b79ed23df,36049,1733316386472 2024-12-04T12:46:32,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2305): Starting mini mapreduce cluster... 2024-12-04T12:46:32,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/test.cache.data in system properties and HBase conf 2024-12-04T12:46:32,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T12:46:32,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir in system properties and HBase conf 2024-12-04T12:46:32,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T12:46:32,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T12:46:32,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T12:46:32,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T12:46:32,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T12:46:32,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T12:46:32,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T12:46:32,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T12:46:32,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T12:46:32,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T12:46:32,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T12:46:32,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T12:46:32,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/nfs.dump.dir in system properties and HBase conf 2024-12-04T12:46:32,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/java.io.tmpdir in system properties and HBase conf 2024-12-04T12:46:32,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T12:46:32,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T12:46:32,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T12:46:32,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741835_1011 (size=592039) 2024-12-04T12:46:32,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741836_1012 (size=1663647) 2024-12-04T12:46:34,427 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T12:46:34,495 WARN [Thread-251 {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T12:46:34,696 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T12:46:34,696 WARN [Thread-251 {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-12-04T12:46:34,697 INFO [Thread-251 {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T12:46:34,698 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T12:46:34,698 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T12:46:34,698 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T12:46:34,701 INFO [Thread-251 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T12:46:34,701 INFO [Thread-251 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T12:46:34,701 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T12:46:34,701 INFO [Thread-251 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T12:46:34,703 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a8eed8e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,AVAILABLE} 2024-12-04T12:46:34,703 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75c0ec4a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-04T12:46:34,712 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ef40a45{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,AVAILABLE} 2024-12-04T12:46:34,712 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336bed82{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-04T12:46:34,813 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2024-12-04T12:46:34,813 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2024-12-04T12:46:34,813 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-12-04T12:46:34,815 INFO [Thread-251 {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-12-04T12:46:34,855 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T12:46:34,892 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T12:46:35,077 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2024-12-04T12:46:35,098 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@50e4b420{jobhistory,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/java.io.tmpdir/jetty-localhost-39257-hadoop-yarn-common-3_4_1_jar-_-any-16200801145981030426/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-12-04T12:46:35,098 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@395940ac{cluster,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/java.io.tmpdir/jetty-localhost-43233-hadoop-yarn-common-3_4_1_jar-_-any-17523997893959098812/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-12-04T12:46:35,099 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3f7727e1{HTTP/1.1, (http/1.1)}{localhost:43233} 2024-12-04T12:46:35,099 INFO [Thread-251 {}] server.AbstractConnector(333): Started ServerConnector@417ea536{HTTP/1.1, (http/1.1)}{localhost:39257} 2024-12-04T12:46:35,099 INFO [Time-limited test {}] server.Server(415): Started @14118ms 2024-12-04T12:46:35,099 INFO [Thread-251 {}] server.Server(415): Started @14118ms 2024-12-04T12:46:35,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741837_1013 (size=5) 2024-12-04T12:46:36,032 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-12-04T12:46:36,038 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T12:46:36,069 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-12-04T12:46:36,070 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T12:46:36,076 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T12:46:36,077 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T12:46:36,077 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T12:46:36,086 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T12:46:36,087 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51553806{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,AVAILABLE} 2024-12-04T12:46:36,087 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6c8395ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-04T12:46:36,134 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-12-04T12:46:36,134 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-12-04T12:46:36,134 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-12-04T12:46:36,134 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-12-04T12:46:36,142 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T12:46:36,148 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T12:46:36,241 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T12:46:36,253 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3860df07{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/java.io.tmpdir/jetty-localhost-39751-hadoop-yarn-common-3_4_1_jar-_-any-17979080056540548058/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-04T12:46:36,254 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1ed8df47{HTTP/1.1, (http/1.1)}{localhost:39751} 2024-12-04T12:46:36,254 INFO [Time-limited test {}] server.Server(415): Started @15273ms 2024-12-04T12:46:36,396 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-12-04T12:46:36,398 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T12:46:36,410 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-12-04T12:46:36,410 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T12:46:36,417 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T12:46:36,417 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T12:46:36,417 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T12:46:36,418 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T12:46:36,418 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@625a811e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,AVAILABLE} 2024-12-04T12:46:36,419 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1edcf525{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-04T12:46:36,464 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-12-04T12:46:36,464 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-12-04T12:46:36,464 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-12-04T12:46:36,465 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-12-04T12:46:36,472 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T12:46:36,479 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T12:46:36,571 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T12:46:36,576 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4a556c09{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/java.io.tmpdir/jetty-localhost-45529-hadoop-yarn-common-3_4_1_jar-_-any-4026443125427828179/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-04T12:46:36,577 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4c619d4c{HTTP/1.1, (http/1.1)}{localhost:45529} 2024-12-04T12:46:36,577 INFO [Time-limited test {}] server.Server(415): Started @15596ms 2024-12-04T12:46:36,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2341): Mini mapreduce cluster started 2024-12-04T12:46:36,597 INFO [Time-limited test {}] backup.TestBackupBase(336): ROOTDIR hdfs://localhost:41555/backupUT 2024-12-04T12:46:36,601 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:46:36,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:46:36,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3ceb4c0b 2024-12-04T12:46:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:46:36,611 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:46:36,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns1'} 2024-12-04T12:46:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns1 2024-12-04T12:46:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T12:46:36,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns1 in 21 msec 2024-12-04T12:46:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T12:46:36,736 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns1 completed 2024-12-04T12:46:36,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns2'} 2024-12-04T12:46:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns2 2024-12-04T12:46:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-12-04T12:46:36,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns2 in 12 msec 2024-12-04T12:46:36,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T12:46:36,831 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T12:46:36,839 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:46:36,839 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2024-12-04T12:46:36,840 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T12:46:36,840 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T12:46:36,840 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T12:46:36,840 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T12:46:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-12-04T12:46:36,855 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns2 completed 2024-12-04T12:46:36,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns3'} 2024-12-04T12:46:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=6, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns3 2024-12-04T12:46:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-12-04T12:46:36,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns3 in 12 msec 2024-12-04T12:46:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-12-04T12:46:36,965 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns3 completed 2024-12-04T12:46:36,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns4'} 2024-12-04T12:46:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns4 2024-12-04T12:46:36,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T12:46:36,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns4 in 12 msec 2024-12-04T12:46:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T12:46:37,075 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns4 completed 2024-12-04T12:46:37,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test-1733316396597', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:46:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test-1733316396597 2024-12-04T12:46:37,093 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test-1733316396597 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T12:46:37,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test-1733316396597" procId is: 8 2024-12-04T12:46:37,096 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-04T12:46:37,099 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test-1733316396597 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T12:46:37,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741838_1014 (size=367) 2024-12-04T12:46:37,143 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T12:46:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-04T12:46:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-04T12:46:37,513 INFO [RegionOpenAndInit-test-1733316396597-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 87e3e36a673c628af1440164c1fd3443, NAME => 'test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1733316396597', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:46:37,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741839_1015 (size=53) 2024-12-04T12:46:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-04T12:46:37,732 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-12-04T12:46:37,923 DEBUG [RegionOpenAndInit-test-1733316396597-pool-0 {}] regionserver.HRegion(898): Instantiated test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:37,923 DEBUG [RegionOpenAndInit-test-1733316396597-pool-0 {}] regionserver.HRegion(1722): Closing 87e3e36a673c628af1440164c1fd3443, disabling compactions & flushes 2024-12-04T12:46:37,923 INFO [RegionOpenAndInit-test-1733316396597-pool-0 {}] regionserver.HRegion(1755): Closing region test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:37,923 DEBUG [RegionOpenAndInit-test-1733316396597-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:37,923 DEBUG [RegionOpenAndInit-test-1733316396597-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. after waiting 0 ms 2024-12-04T12:46:37,923 DEBUG [RegionOpenAndInit-test-1733316396597-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:37,924 INFO [RegionOpenAndInit-test-1733316396597-pool-0 {}] regionserver.HRegion(1973): Closed test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:37,924 DEBUG [RegionOpenAndInit-test-1733316396597-pool-0 {}] regionserver.HRegion(1676): Region close journal for 87e3e36a673c628af1440164c1fd3443: Waiting for close lock at 1733316397923Disabling compacts and flushes for region at 1733316397923Disabling writes for close at 1733316397923Writing region close event to WAL at 1733316397923Closed at 1733316397923 2024-12-04T12:46:37,926 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test-1733316396597 execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T12:46:37,930 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733316397926"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733316397926"}]},"ts":"1733316397926"} 2024-12-04T12:46:37,936 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T12:46:37,938 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test-1733316396597 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T12:46:37,941 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1733316396597","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316397938"}]},"ts":"1733316397938"} 2024-12-04T12:46:37,946 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1733316396597, state=ENABLING in hbase:meta 2024-12-04T12:46:37,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, ASSIGN}] 2024-12-04T12:46:37,950 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, ASSIGN 2024-12-04T12:46:37,951 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, ASSIGN; state=OFFLINE, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=false 2024-12-04T12:46:38,103 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=87e3e36a673c628af1440164c1fd3443, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:38,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, ASSIGN because future has completed 2024-12-04T12:46:38,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87e3e36a673c628af1440164c1fd3443, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:46:38,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-04T12:46:38,284 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(132): Open test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:38,284 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T12:46:38,285 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] monitor.StreamSlowMonitor(122): New stream slow monitor 938b79ed23df,38879,1733316387992.regiongroup-1 2024-12-04T12:46:38,287 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1, suffix=, logDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992, archiveDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs, maxLogs=32 2024-12-04T12:46:38,300 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316398288, exclude list is [], retry=0 2024-12-04T12:46:38,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:46:38,306 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316398288 2024-12-04T12:46:38,306 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:46:38,307 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7752): Opening region: {ENCODED => 87e3e36a673c628af1440164c1fd3443, NAME => 'test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443.', STARTKEY => '', ENDKEY => ''} 2024-12-04T12:46:38,307 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:46:38,307 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1733316396597 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:38,307 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(898): Instantiated test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:38,307 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7794): checking encryption for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:38,307 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7797): checking classloading for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:38,310 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:38,312 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87e3e36a673c628af1440164c1fd3443 columnFamilyName f 2024-12-04T12:46:38,312 DEBUG [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:38,313 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.HStore(327): Store=87e3e36a673c628af1440164c1fd3443/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:46:38,314 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1038): replaying wal for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:38,315 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:38,316 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:38,316 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1048): stopping wal replay for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:38,317 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1060): Cleaning up temporary data for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:38,320 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1093): writing seq id for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:38,324 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T12:46:38,325 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1114): Opened 87e3e36a673c628af1440164c1fd3443; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9483205120, jitterRate=-0.11680769920349121}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T12:46:38,325 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:38,326 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1006): Region open journal for 87e3e36a673c628af1440164c1fd3443: Running coprocessor pre-open hook at 1733316398308Writing region info on filesystem at 1733316398308Initializing all the Stores at 1733316398309 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316398309Cleaning up temporary data from old regions at 1733316398317 (+8 ms)Running coprocessor post-open hooks at 1733316398325 (+8 ms)Region opened successfully at 1733316398326 (+1 ms) 2024-12-04T12:46:38,328 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443., pid=10, masterSystemTime=1733316398262 2024-12-04T12:46:38,331 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:38,332 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(153): Opened test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:38,333 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=87e3e36a673c628af1440164c1fd3443, regionState=OPEN, openSeqNum=2, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:38,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87e3e36a673c628af1440164c1fd3443, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:46:38,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-04T12:46:38,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 87e3e36a673c628af1440164c1fd3443, server=938b79ed23df,38879,1733316387992 in 231 msec 2024-12-04T12:46:38,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-04T12:46:38,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, ASSIGN in 397 msec 2024-12-04T12:46:38,351 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test-1733316396597 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T12:46:38,352 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1733316396597","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316398351"}]},"ts":"1733316398351"} 2024-12-04T12:46:38,356 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1733316396597, state=ENABLED in hbase:meta 2024-12-04T12:46:38,358 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test-1733316396597 execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T12:46:38,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; CreateTableProcedure table=test-1733316396597 in 1.2720 sec 2024-12-04T12:46:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-04T12:46:39,245 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test-1733316396597 completed 2024-12-04T12:46:39,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5bc42e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:39,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:46:39,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:46:39,248 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:46:39,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:46:39,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:46:39,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251422ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:39,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:46:39,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:46:39,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:46:39,251 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:46:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e2885ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:39,255 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:46:39,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:46:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:46:39,260 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:46:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733316396597', row='row0', locateType=CURRENT is [region=test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:46:39,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HRegion(8528): writing data to region test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. with WAL disabled. Data may be lost in the event of a crash. 2024-12-04T12:46:39,432 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns2:test-17333163965971', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:46:39,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:test-17333163965971 2024-12-04T12:46:39,435 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17333163965971 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T12:46:39,435 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:39,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns2" qualifier: "test-17333163965971" procId is: 11 2024-12-04T12:46:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T12:46:39,437 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:test-17333163965971 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T12:46:39,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741841_1017 (size=364) 2024-12-04T12:46:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T12:46:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T12:46:39,848 INFO [RegionOpenAndInit-ns2:test-17333163965971-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c988c84788a01c971b5789026625491a, NAME => 'ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17333163965971', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:46:39,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741842_1018 (size=50) 2024-12-04T12:46:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T12:46:40,258 DEBUG [RegionOpenAndInit-ns2:test-17333163965971-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:40,259 DEBUG [RegionOpenAndInit-ns2:test-17333163965971-pool-0 {}] regionserver.HRegion(1722): Closing c988c84788a01c971b5789026625491a, disabling compactions & flushes 2024-12-04T12:46:40,259 INFO [RegionOpenAndInit-ns2:test-17333163965971-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:46:40,259 DEBUG [RegionOpenAndInit-ns2:test-17333163965971-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:46:40,259 DEBUG [RegionOpenAndInit-ns2:test-17333163965971-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. after waiting 0 ms 2024-12-04T12:46:40,259 DEBUG [RegionOpenAndInit-ns2:test-17333163965971-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:46:40,259 INFO [RegionOpenAndInit-ns2:test-17333163965971-pool-0 {}] regionserver.HRegion(1973): Closed ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:46:40,259 DEBUG [RegionOpenAndInit-ns2:test-17333163965971-pool-0 {}] regionserver.HRegion(1676): Region close journal for c988c84788a01c971b5789026625491a: Waiting for close lock at 1733316400259Disabling compacts and flushes for region at 1733316400259Disabling writes for close at 1733316400259Writing region close event to WAL at 1733316400259Closed at 1733316400259 2024-12-04T12:46:40,261 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:test-17333163965971 execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T12:46:40,261 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1733316400261"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733316400261"}]},"ts":"1733316400261"} 2024-12-04T12:46:40,265 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T12:46:40,266 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:test-17333163965971 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T12:46:40,267 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17333163965971","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316400266"}]},"ts":"1733316400266"} 2024-12-04T12:46:40,269 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17333163965971, state=ENABLING in hbase:meta 2024-12-04T12:46:40,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17333163965971, region=c988c84788a01c971b5789026625491a, ASSIGN}] 2024-12-04T12:46:40,272 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17333163965971, region=c988c84788a01c971b5789026625491a, ASSIGN 2024-12-04T12:46:40,273 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:test-17333163965971, region=c988c84788a01c971b5789026625491a, ASSIGN; state=OFFLINE, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=false 2024-12-04T12:46:40,424 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=c988c84788a01c971b5789026625491a, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:40,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:test-17333163965971, region=c988c84788a01c971b5789026625491a, ASSIGN because future has completed 2024-12-04T12:46:40,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure c988c84788a01c971b5789026625491a, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:46:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T12:46:40,586 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:46:40,587 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => c988c84788a01c971b5789026625491a, NAME => 'ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a.', STARTKEY => '', ENDKEY => ''} 2024-12-04T12:46:40,587 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:46:40,587 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17333163965971 c988c84788a01c971b5789026625491a 2024-12-04T12:46:40,587 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:40,587 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for c988c84788a01c971b5789026625491a 2024-12-04T12:46:40,587 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for c988c84788a01c971b5789026625491a 2024-12-04T12:46:40,589 INFO [StoreOpener-c988c84788a01c971b5789026625491a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region c988c84788a01c971b5789026625491a 2024-12-04T12:46:40,591 INFO [StoreOpener-c988c84788a01c971b5789026625491a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c988c84788a01c971b5789026625491a columnFamilyName f 2024-12-04T12:46:40,592 DEBUG [StoreOpener-c988c84788a01c971b5789026625491a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:40,592 INFO [StoreOpener-c988c84788a01c971b5789026625491a-1 {}] regionserver.HStore(327): Store=c988c84788a01c971b5789026625491a/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:46:40,593 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for c988c84788a01c971b5789026625491a 2024-12-04T12:46:40,594 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a 2024-12-04T12:46:40,595 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a 2024-12-04T12:46:40,596 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for c988c84788a01c971b5789026625491a 2024-12-04T12:46:40,596 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for c988c84788a01c971b5789026625491a 2024-12-04T12:46:40,598 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for c988c84788a01c971b5789026625491a 2024-12-04T12:46:40,602 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T12:46:40,603 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened c988c84788a01c971b5789026625491a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11123853280, jitterRate=0.03598956763744354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T12:46:40,603 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c988c84788a01c971b5789026625491a 2024-12-04T12:46:40,604 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for c988c84788a01c971b5789026625491a: Running coprocessor pre-open hook at 1733316400588Writing region info on filesystem at 1733316400588Initializing all the Stores at 1733316400589 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316400589Cleaning up temporary data from old regions at 1733316400596 (+7 ms)Running coprocessor post-open hooks at 1733316400603 (+7 ms)Region opened successfully at 1733316400604 (+1 ms) 2024-12-04T12:46:40,606 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a., pid=13, masterSystemTime=1733316400581 2024-12-04T12:46:40,609 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:46:40,609 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:46:40,610 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=c988c84788a01c971b5789026625491a, regionState=OPEN, openSeqNum=2, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:40,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure c988c84788a01c971b5789026625491a, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:46:40,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-12-04T12:46:40,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure c988c84788a01c971b5789026625491a, server=938b79ed23df,38879,1733316387992 in 187 msec 2024-12-04T12:46:40,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-04T12:46:40,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:test-17333163965971, region=c988c84788a01c971b5789026625491a, ASSIGN in 349 msec 2024-12-04T12:46:40,625 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:test-17333163965971 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T12:46:40,625 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17333163965971","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316400625"}]},"ts":"1733316400625"} 2024-12-04T12:46:40,628 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17333163965971, state=ENABLED in hbase:meta 2024-12-04T12:46:40,629 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17333163965971 execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T12:46:40,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:test-17333163965971 in 1.1980 sec 2024-12-04T12:46:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T12:46:41,585 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:test-17333163965971 completed 2024-12-04T12:46:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17333163965971', row='row0', locateType=CURRENT is [region=ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:46:41,591 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HRegion(8528): writing data to region ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. with WAL disabled. Data may be lost in the event of a crash. 2024-12-04T12:46:41,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns3:test-17333163965972', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:46:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns3:test-17333163965972 2024-12-04T12:46:41,768 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17333163965972 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T12:46:41,768 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:41,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns3" qualifier: "test-17333163965972" procId is: 14 2024-12-04T12:46:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-04T12:46:41,770 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns3:test-17333163965972 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T12:46:41,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741843_1019 (size=390) 2024-12-04T12:46:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-04T12:46:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-04T12:46:42,182 INFO [RegionOpenAndInit-ns3:test-17333163965972-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d7d9cf8458b0e699dff32ea7d56f7d9d, NAME => 'ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17333163965972', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:46:42,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741844_1020 (size=50) 2024-12-04T12:46:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-04T12:46:42,592 DEBUG [RegionOpenAndInit-ns3:test-17333163965972-pool-0 {}] regionserver.HRegion(898): Instantiated ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:42,593 DEBUG [RegionOpenAndInit-ns3:test-17333163965972-pool-0 {}] regionserver.HRegion(1722): Closing d7d9cf8458b0e699dff32ea7d56f7d9d, disabling compactions & flushes 2024-12-04T12:46:42,593 INFO [RegionOpenAndInit-ns3:test-17333163965972-pool-0 {}] regionserver.HRegion(1755): Closing region ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. 2024-12-04T12:46:42,593 DEBUG [RegionOpenAndInit-ns3:test-17333163965972-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. 2024-12-04T12:46:42,593 DEBUG [RegionOpenAndInit-ns3:test-17333163965972-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. after waiting 0 ms 2024-12-04T12:46:42,593 DEBUG [RegionOpenAndInit-ns3:test-17333163965972-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. 2024-12-04T12:46:42,593 INFO [RegionOpenAndInit-ns3:test-17333163965972-pool-0 {}] regionserver.HRegion(1973): Closed ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. 2024-12-04T12:46:42,593 DEBUG [RegionOpenAndInit-ns3:test-17333163965972-pool-0 {}] regionserver.HRegion(1676): Region close journal for d7d9cf8458b0e699dff32ea7d56f7d9d: Waiting for close lock at 1733316402593Disabling compacts and flushes for region at 1733316402593Disabling writes for close at 1733316402593Writing region close event to WAL at 1733316402593Closed at 1733316402593 2024-12-04T12:46:42,595 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns3:test-17333163965972 execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T12:46:42,596 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1733316402595"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733316402595"}]},"ts":"1733316402595"} 2024-12-04T12:46:42,599 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T12:46:42,601 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns3:test-17333163965972 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T12:46:42,602 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17333163965972","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316402601"}]},"ts":"1733316402601"} 2024-12-04T12:46:42,605 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17333163965972, state=ENABLING in hbase:meta 2024-12-04T12:46:42,606 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17333163965972, region=d7d9cf8458b0e699dff32ea7d56f7d9d, ASSIGN}] 2024-12-04T12:46:42,612 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17333163965972, region=d7d9cf8458b0e699dff32ea7d56f7d9d, ASSIGN 2024-12-04T12:46:42,614 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns3:test-17333163965972, region=d7d9cf8458b0e699dff32ea7d56f7d9d, ASSIGN; state=OFFLINE, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=false 2024-12-04T12:46:42,765 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=d7d9cf8458b0e699dff32ea7d56f7d9d, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:42,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns3:test-17333163965972, region=d7d9cf8458b0e699dff32ea7d56f7d9d, ASSIGN because future has completed 2024-12-04T12:46:42,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7d9cf8458b0e699dff32ea7d56f7d9d, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:46:42,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-04T12:46:42,929 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. 2024-12-04T12:46:42,929 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => d7d9cf8458b0e699dff32ea7d56f7d9d, NAME => 'ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d.', STARTKEY => '', ENDKEY => ''} 2024-12-04T12:46:42,930 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:46:42,930 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17333163965972 d7d9cf8458b0e699dff32ea7d56f7d9d 2024-12-04T12:46:42,930 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:42,930 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for d7d9cf8458b0e699dff32ea7d56f7d9d 2024-12-04T12:46:42,931 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for d7d9cf8458b0e699dff32ea7d56f7d9d 2024-12-04T12:46:42,933 INFO [StoreOpener-d7d9cf8458b0e699dff32ea7d56f7d9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d7d9cf8458b0e699dff32ea7d56f7d9d 2024-12-04T12:46:42,936 INFO [StoreOpener-d7d9cf8458b0e699dff32ea7d56f7d9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7d9cf8458b0e699dff32ea7d56f7d9d columnFamilyName f 2024-12-04T12:46:42,936 DEBUG [StoreOpener-d7d9cf8458b0e699dff32ea7d56f7d9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:42,937 INFO [StoreOpener-d7d9cf8458b0e699dff32ea7d56f7d9d-1 {}] regionserver.HStore(327): Store=d7d9cf8458b0e699dff32ea7d56f7d9d/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:46:42,938 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for d7d9cf8458b0e699dff32ea7d56f7d9d 2024-12-04T12:46:42,939 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns3/test-17333163965972/d7d9cf8458b0e699dff32ea7d56f7d9d 2024-12-04T12:46:42,940 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns3/test-17333163965972/d7d9cf8458b0e699dff32ea7d56f7d9d 2024-12-04T12:46:42,941 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for d7d9cf8458b0e699dff32ea7d56f7d9d 2024-12-04T12:46:42,941 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for d7d9cf8458b0e699dff32ea7d56f7d9d 2024-12-04T12:46:42,945 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for d7d9cf8458b0e699dff32ea7d56f7d9d 2024-12-04T12:46:42,949 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns3/test-17333163965972/d7d9cf8458b0e699dff32ea7d56f7d9d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T12:46:42,950 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened d7d9cf8458b0e699dff32ea7d56f7d9d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9652482880, jitterRate=-0.1010424792766571}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T12:46:42,950 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d7d9cf8458b0e699dff32ea7d56f7d9d 2024-12-04T12:46:42,952 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for d7d9cf8458b0e699dff32ea7d56f7d9d: Running coprocessor pre-open hook at 1733316402931Writing region info on filesystem at 1733316402931Initializing all the Stores at 1733316402933 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316402933Cleaning up temporary data from old regions at 1733316402941 (+8 ms)Running coprocessor post-open hooks at 1733316402951 (+10 ms)Region opened successfully at 1733316402952 (+1 ms) 2024-12-04T12:46:42,953 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d., pid=16, masterSystemTime=1733316402923 2024-12-04T12:46:42,957 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. 2024-12-04T12:46:42,957 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. 2024-12-04T12:46:42,958 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=d7d9cf8458b0e699dff32ea7d56f7d9d, regionState=OPEN, openSeqNum=2, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:42,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7d9cf8458b0e699dff32ea7d56f7d9d, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:46:42,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-04T12:46:42,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure d7d9cf8458b0e699dff32ea7d56f7d9d, server=938b79ed23df,38879,1733316387992 in 195 msec 2024-12-04T12:46:42,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-04T12:46:42,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns3:test-17333163965972, region=d7d9cf8458b0e699dff32ea7d56f7d9d, ASSIGN in 363 msec 2024-12-04T12:46:42,973 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns3:test-17333163965972 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T12:46:42,974 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17333163965972","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316402974"}]},"ts":"1733316402974"} 2024-12-04T12:46:42,977 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17333163965972, state=ENABLED in hbase:meta 2024-12-04T12:46:42,979 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17333163965972 execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T12:46:42,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns3:test-17333163965972 in 1.2150 sec 2024-12-04T12:46:43,146 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17333163965972' 2024-12-04T12:46:43,146 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1733316396597' 2024-12-04T12:46:43,147 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17333163965971' 2024-12-04T12:46:43,810 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-12-04T12:46:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-04T12:46:43,914 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns3:test-17333163965972 completed 2024-12-04T12:46:43,915 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns3:test-17333163965972 get assigned. Timeout = 60000ms 2024-12-04T12:46:43,917 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T12:46:43,922 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns3:test-17333163965972 assigned to meta. Checking AM states. 2024-12-04T12:46:43,922 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T12:46:43,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns3:test-17333163965972 assigned. 2024-12-04T12:46:43,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns4:test-17333163965973', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:46:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns4:test-17333163965973 2024-12-04T12:46:43,929 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17333163965973 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T12:46:43,929 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:43,929 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns4" qualifier: "test-17333163965973" procId is: 17 2024-12-04T12:46:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T12:46:43,931 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns4:test-17333163965973 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T12:46:43,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741845_1021 (size=390) 2024-12-04T12:46:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T12:46:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T12:46:44,343 INFO [RegionOpenAndInit-ns4:test-17333163965973-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => deae8c07fc7147a6340e830c87bd97da, NAME => 'ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17333163965973', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:46:44,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741846_1022 (size=50) 2024-12-04T12:46:44,352 DEBUG [RegionOpenAndInit-ns4:test-17333163965973-pool-0 {}] regionserver.HRegion(898): Instantiated ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:44,352 DEBUG [RegionOpenAndInit-ns4:test-17333163965973-pool-0 {}] regionserver.HRegion(1722): Closing deae8c07fc7147a6340e830c87bd97da, disabling compactions & flushes 2024-12-04T12:46:44,353 INFO [RegionOpenAndInit-ns4:test-17333163965973-pool-0 {}] regionserver.HRegion(1755): Closing region ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. 2024-12-04T12:46:44,353 DEBUG [RegionOpenAndInit-ns4:test-17333163965973-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. 2024-12-04T12:46:44,353 DEBUG [RegionOpenAndInit-ns4:test-17333163965973-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. after waiting 0 ms 2024-12-04T12:46:44,353 DEBUG [RegionOpenAndInit-ns4:test-17333163965973-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. 2024-12-04T12:46:44,353 INFO [RegionOpenAndInit-ns4:test-17333163965973-pool-0 {}] regionserver.HRegion(1973): Closed ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. 2024-12-04T12:46:44,353 DEBUG [RegionOpenAndInit-ns4:test-17333163965973-pool-0 {}] regionserver.HRegion(1676): Region close journal for deae8c07fc7147a6340e830c87bd97da: Waiting for close lock at 1733316404352Disabling compacts and flushes for region at 1733316404352Disabling writes for close at 1733316404353 (+1 ms)Writing region close event to WAL at 1733316404353Closed at 1733316404353 2024-12-04T12:46:44,355 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns4:test-17333163965973 execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T12:46:44,355 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1733316404355"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733316404355"}]},"ts":"1733316404355"} 2024-12-04T12:46:44,358 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T12:46:44,360 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns4:test-17333163965973 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T12:46:44,360 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17333163965973","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316404360"}]},"ts":"1733316404360"} 2024-12-04T12:46:44,363 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17333163965973, state=ENABLING in hbase:meta 2024-12-04T12:46:44,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17333163965973, region=deae8c07fc7147a6340e830c87bd97da, ASSIGN}] 2024-12-04T12:46:44,366 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17333163965973, region=deae8c07fc7147a6340e830c87bd97da, ASSIGN 2024-12-04T12:46:44,367 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns4:test-17333163965973, region=deae8c07fc7147a6340e830c87bd97da, ASSIGN; state=OFFLINE, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=false 2024-12-04T12:46:44,518 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=deae8c07fc7147a6340e830c87bd97da, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:44,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns4:test-17333163965973, region=deae8c07fc7147a6340e830c87bd97da, ASSIGN because future has completed 2024-12-04T12:46:44,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure deae8c07fc7147a6340e830c87bd97da, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:46:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T12:46:44,680 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. 2024-12-04T12:46:44,680 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => deae8c07fc7147a6340e830c87bd97da, NAME => 'ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da.', STARTKEY => '', ENDKEY => ''} 2024-12-04T12:46:44,681 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:46:44,681 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17333163965973 deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:46:44,681 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:44,681 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:46:44,681 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:46:44,683 INFO [StoreOpener-deae8c07fc7147a6340e830c87bd97da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:46:44,686 INFO [StoreOpener-deae8c07fc7147a6340e830c87bd97da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region deae8c07fc7147a6340e830c87bd97da columnFamilyName f 2024-12-04T12:46:44,686 DEBUG [StoreOpener-deae8c07fc7147a6340e830c87bd97da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:44,686 INFO [StoreOpener-deae8c07fc7147a6340e830c87bd97da-1 {}] regionserver.HStore(327): Store=deae8c07fc7147a6340e830c87bd97da/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:46:44,687 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:46:44,687 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns4/test-17333163965973/deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:46:44,688 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns4/test-17333163965973/deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:46:44,689 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:46:44,689 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:46:44,691 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:46:44,694 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns4/test-17333163965973/deae8c07fc7147a6340e830c87bd97da/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T12:46:44,695 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened deae8c07fc7147a6340e830c87bd97da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11719952960, jitterRate=0.09150567650794983}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T12:46:44,695 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:46:44,696 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for deae8c07fc7147a6340e830c87bd97da: Running coprocessor pre-open hook at 1733316404681Writing region info on filesystem at 1733316404681Initializing all the Stores at 1733316404683 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316404683Cleaning up temporary data from old regions at 1733316404689 (+6 ms)Running coprocessor post-open hooks at 1733316404695 (+6 ms)Region opened successfully at 1733316404696 (+1 ms) 2024-12-04T12:46:44,697 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da., pid=19, masterSystemTime=1733316404675 2024-12-04T12:46:44,700 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. 2024-12-04T12:46:44,700 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. 2024-12-04T12:46:44,701 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=deae8c07fc7147a6340e830c87bd97da, regionState=OPEN, openSeqNum=2, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:44,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure deae8c07fc7147a6340e830c87bd97da, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:46:44,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-04T12:46:44,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure deae8c07fc7147a6340e830c87bd97da, server=938b79ed23df,38879,1733316387992 in 184 msec 2024-12-04T12:46:44,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-04T12:46:44,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns4:test-17333163965973, region=deae8c07fc7147a6340e830c87bd97da, ASSIGN in 345 msec 2024-12-04T12:46:44,714 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns4:test-17333163965973 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T12:46:44,714 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17333163965973","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316404714"}]},"ts":"1733316404714"} 2024-12-04T12:46:44,717 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17333163965973, state=ENABLED in hbase:meta 2024-12-04T12:46:44,718 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17333163965973 execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T12:46:44,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns4:test-17333163965973 in 794 msec 2024-12-04T12:46:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T12:46:45,066 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns4:test-17333163965973 completed 2024-12-04T12:46:45,066 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns4:test-17333163965973 get assigned. Timeout = 60000ms 2024-12-04T12:46:45,066 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T12:46:45,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns4:test-17333163965973 assigned to meta. Checking AM states. 2024-12-04T12:46:45,077 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T12:46:45,077 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns4:test-17333163965973 assigned. 2024-12-04T12:46:45,077 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:46:45,078 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestBackupBase.createTables(TestBackupBase.java:481) at org.apache.hadoop.hbase.backup.TestBackupBase.setUpHelper(TestBackupBase.java:343) at org.apache.hadoop.hbase.backup.TestBackupBase.setUp(TestBackupBase.java:357) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:46:45,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:46:45,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:46:45,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:46:45,105 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=611, OpenFileDescriptor=633, MaxFileDescriptor=1048576, SystemLoadAverage=122, ProcessCount=14, AvailableMemoryMB=7253 2024-12-04T12:46:45,107 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=611 is superior to 500 2024-12-04T12:46:45,108 INFO [Time-limited test {}] backup.TestIncrementalBackup(91): create full backup image for all tables 2024-12-04T12:46:45,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table test-1733316396597 from 'test-1733316396597', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1733316396597', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:46:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=20, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1733316396597 2024-12-04T12:46:45,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741847_1023 (size=908) 2024-12-04T12:46:45,537 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/.tabledesc/.tableinfo.0000000001.367 2024-12-04T12:46:45,537 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/.tabledesc/.tableinfo.0000000002.908 2024-12-04T12:46:45,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1733316396597}] 2024-12-04T12:46:45,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, REOPEN/MOVE}] 2024-12-04T12:46:45,553 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, REOPEN/MOVE 2024-12-04T12:46:45,555 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=87e3e36a673c628af1440164c1fd3443, regionState=CLOSING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:45,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, REOPEN/MOVE because future has completed 2024-12-04T12:46:45,559 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T12:46:45,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 87e3e36a673c628af1440164c1fd3443, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:46:45,717 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(122): Close 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:45,717 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T12:46:45,718 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1722): Closing 87e3e36a673c628af1440164c1fd3443, disabling compactions & flushes 2024-12-04T12:46:45,718 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1755): Closing region test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:45,718 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:45,718 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1843): Acquired close lock on test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. after waiting 0 ms 2024-12-04T12:46:45,718 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1853): Updates disabled for region test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:45,722 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(2902): Flushing 87e3e36a673c628af1440164c1fd3443 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-12-04T12:46:45,790 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/f/de63a3baba294af8906a08a560deeb40 is 37, key is row10/f:q1/1733316399288/Put/seqid=0 2024-12-04T12:46:45,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741848_1024 (size=8413) 2024-12-04T12:46:46,201 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/f/de63a3baba294af8906a08a560deeb40 2024-12-04T12:46:46,274 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/f/de63a3baba294af8906a08a560deeb40 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40 2024-12-04T12:46:46,284 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40, entries=99, sequenceid=103, filesize=8.2 K 2024-12-04T12:46:46,291 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 87e3e36a673c628af1440164c1fd3443 in 568ms, sequenceid=103, compaction requested=false 2024-12-04T12:46:46,299 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2024-12-04T12:46:46,302 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:46:46,302 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1973): Closed test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:46,302 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1676): Region close journal for 87e3e36a673c628af1440164c1fd3443: Waiting for close lock at 1733316405718Running coprocessor pre-close hooks at 1733316405718Disabling compacts and flushes for region at 1733316405718Disabling writes for close at 1733316405718Obtaining lock to block concurrent updates at 1733316405722 (+4 ms)Preparing flush snapshotting stores in 87e3e36a673c628af1440164c1fd3443 at 1733316405722Finished memstore snapshotting test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443., syncing WAL and waiting on mvcc, flushsize=dataSize=3247, getHeapSize=11248, getOffHeapSize=0, getCellsCount=99 at 1733316405730 (+8 ms)Flushing stores of test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. at 1733316405732 (+2 ms)Flushing 87e3e36a673c628af1440164c1fd3443/f: creating writer at 1733316405734 (+2 ms)Flushing 87e3e36a673c628af1440164c1fd3443/f: appending metadata at 1733316405776 (+42 ms)Flushing 87e3e36a673c628af1440164c1fd3443/f: closing flushed file at 1733316405780 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fcb7e4b: reopening flushed file at 1733316406272 (+492 ms)Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 87e3e36a673c628af1440164c1fd3443 in 568ms, sequenceid=103, compaction requested=false at 1733316406291 (+19 ms)Writing region close event to WAL at 1733316406293 (+2 ms)Running coprocessor post-close hooks at 1733316406300 (+7 ms)Closed at 1733316406302 (+2 ms) 2024-12-04T12:46:46,302 WARN [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionServer(3299): Not adding moved region record: 87e3e36a673c628af1440164c1fd3443 to self. 2024-12-04T12:46:46,306 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(157): Closed 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:46,307 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=87e3e36a673c628af1440164c1fd3443, regionState=CLOSED 2024-12-04T12:46:46,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 87e3e36a673c628af1440164c1fd3443, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:46:46,315 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-04T12:46:46,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; CloseRegionProcedure 87e3e36a673c628af1440164c1fd3443, server=938b79ed23df,38879,1733316387992 in 752 msec 2024-12-04T12:46:46,316 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, REOPEN/MOVE; state=CLOSED, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=true 2024-12-04T12:46:46,467 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=87e3e36a673c628af1440164c1fd3443, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:46,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, REOPEN/MOVE because future has completed 2024-12-04T12:46:46,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87e3e36a673c628af1440164c1fd3443, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:46:46,633 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:46,634 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 87e3e36a673c628af1440164c1fd3443, NAME => 'test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443.', STARTKEY => '', ENDKEY => ''} 2024-12-04T12:46:46,634 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:46:46,635 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1733316396597 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:46,635 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:46,635 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:46,635 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:46,638 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:46,639 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87e3e36a673c628af1440164c1fd3443 columnFamilyName f 2024-12-04T12:46:46,640 DEBUG [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:46,649 DEBUG [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40 2024-12-04T12:46:46,650 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.HStore(327): Store=87e3e36a673c628af1440164c1fd3443/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:46:46,650 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:46,652 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87e3e36a673c628af1440164c1fd3443 columnFamilyName f3 2024-12-04T12:46:46,652 DEBUG [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:46,653 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.HStore(327): Store=87e3e36a673c628af1440164c1fd3443/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:46:46,654 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:46,660 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87e3e36a673c628af1440164c1fd3443 columnFamilyName mob 2024-12-04T12:46:46,665 DEBUG [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:46,666 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.HStore(327): Store=87e3e36a673c628af1440164c1fd3443/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:46:46,667 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:46,668 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:46,670 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:46,671 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:46,671 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:46,672 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1733316396597 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T12:46:46,674 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:46,675 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 87e3e36a673c628af1440164c1fd3443; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11710806080, jitterRate=0.09065380692481995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T12:46:46,675 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:46,676 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 87e3e36a673c628af1440164c1fd3443: Running coprocessor pre-open hook at 1733316406635Writing region info on filesystem at 1733316406635Initializing all the Stores at 1733316406637 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316406638 (+1 ms)Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316406638Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316406638Cleaning up temporary data from old regions at 1733316406671 (+33 ms)Running coprocessor post-open hooks at 1733316406675 (+4 ms)Region opened successfully at 1733316406676 (+1 ms) 2024-12-04T12:46:46,677 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443., pid=24, masterSystemTime=1733316406623 2024-12-04T12:46:46,679 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:46,680 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:46,680 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=87e3e36a673c628af1440164c1fd3443, regionState=OPEN, openSeqNum=107, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:46,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87e3e36a673c628af1440164c1fd3443, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:46:46,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22 2024-12-04T12:46:46,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 87e3e36a673c628af1440164c1fd3443, server=938b79ed23df,38879,1733316387992 in 214 msec 2024-12-04T12:46:46,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-04T12:46:46,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, REOPEN/MOVE in 1.1350 sec 2024-12-04T12:46:46,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-04T12:46:46,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1733316396597 in 1.1500 sec 2024-12-04T12:46:46,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1733316396597 in 1.5820 sec 2024-12-04T12:46:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=20 2024-12-04T12:46:46,701 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1733316396597 completed 2024-12-04T12:46:46,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582cf7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:46,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:46:46,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:46:46,703 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:46:46,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:46:46,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:46:46,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c17bcd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:46,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:46:46,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:46:46,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:46:46,705 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:46:46,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@588e8d9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:46,707 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:46:46,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:46:46,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:46:46,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:46:46,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733316396597', row='row-test-1733316396597-3-0', locateType=CURRENT is [region=test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443., hostname=938b79ed23df,38879,1733316387992, seqNum=107] 2024-12-04T12:46:46,756 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:46:46,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:46:46,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@36b72492 2024-12-04T12:46:46,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:46:46,760 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:46:46,772 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'backup'} 2024-12-04T12:46:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=backup 2024-12-04T12:46:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T12:46:46,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=backup in 9 msec 2024-12-04T12:46:46,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1733316396597 2024-12-04T12:46:46,830 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1733316396597 Metrics about Tables on a single HBase RegionServer 2024-12-04T12:46:46,831 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17333163965973 2024-12-04T12:46:46,831 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17333163965973 Metrics about Tables on a single HBase RegionServer 2024-12-04T12:46:46,831 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17333163965971 2024-12-04T12:46:46,831 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17333163965971 Metrics about Tables on a single HBase RegionServer 2024-12-04T12:46:46,832 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17333163965972 2024-12-04T12:46:46,832 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17333163965972 Metrics about Tables on a single HBase RegionServer 2024-12-04T12:46:46,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T12:46:46,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: backup completed 2024-12-04T12:46:46,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:46:46,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system 2024-12-04T12:46:46,903 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T12:46:46,903 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:46,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2024-12-04T12:46:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T12:46:46,904 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T12:46:46,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741849_1025 (size=635) 2024-12-04T12:46:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T12:46:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T12:46:47,317 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9dab1d763bdadc4adf3b254f1a913949, NAME => 'backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:46:47,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741850_1026 (size=40) 2024-12-04T12:46:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T12:46:47,731 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:47,731 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1722): Closing 9dab1d763bdadc4adf3b254f1a913949, disabling compactions & flushes 2024-12-04T12:46:47,731 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:46:47,731 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:46:47,731 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. after waiting 0 ms 2024-12-04T12:46:47,731 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:46:47,731 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1973): Closed backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:46:47,732 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9dab1d763bdadc4adf3b254f1a913949: Waiting for close lock at 1733316407731Disabling compacts and flushes for region at 1733316407731Disabling writes for close at 1733316407731Writing region close event to WAL at 1733316407731Closed at 1733316407731 2024-12-04T12:46:47,735 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T12:46:47,736 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1733316407735"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733316407735"}]},"ts":"1733316407735"} 2024-12-04T12:46:47,739 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T12:46:47,740 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T12:46:47,741 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316407741"}]},"ts":"1733316407741"} 2024-12-04T12:46:47,743 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-12-04T12:46:47,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9dab1d763bdadc4adf3b254f1a913949, ASSIGN}] 2024-12-04T12:46:47,745 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9dab1d763bdadc4adf3b254f1a913949, ASSIGN 2024-12-04T12:46:47,746 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=9dab1d763bdadc4adf3b254f1a913949, ASSIGN; state=OFFLINE, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=false 2024-12-04T12:46:47,897 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9dab1d763bdadc4adf3b254f1a913949, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:47,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=9dab1d763bdadc4adf3b254f1a913949, ASSIGN because future has completed 2024-12-04T12:46:47,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9dab1d763bdadc4adf3b254f1a913949, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:46:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T12:46:48,069 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:46:48,069 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 9dab1d763bdadc4adf3b254f1a913949, NAME => 'backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949.', STARTKEY => '', ENDKEY => ''} 2024-12-04T12:46:48,070 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:46:48,070 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:46:48,070 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:48,071 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:46:48,071 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:46:48,073 INFO [StoreOpener-9dab1d763bdadc4adf3b254f1a913949-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:46:48,076 INFO [StoreOpener-9dab1d763bdadc4adf3b254f1a913949-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9dab1d763bdadc4adf3b254f1a913949 columnFamilyName meta 2024-12-04T12:46:48,076 DEBUG [StoreOpener-9dab1d763bdadc4adf3b254f1a913949-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:48,078 INFO [StoreOpener-9dab1d763bdadc4adf3b254f1a913949-1 {}] regionserver.HStore(327): Store=9dab1d763bdadc4adf3b254f1a913949/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:46:48,078 INFO [StoreOpener-9dab1d763bdadc4adf3b254f1a913949-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:46:48,081 INFO [StoreOpener-9dab1d763bdadc4adf3b254f1a913949-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9dab1d763bdadc4adf3b254f1a913949 columnFamilyName session 2024-12-04T12:46:48,081 DEBUG [StoreOpener-9dab1d763bdadc4adf3b254f1a913949-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:48,082 INFO [StoreOpener-9dab1d763bdadc4adf3b254f1a913949-1 {}] regionserver.HStore(327): Store=9dab1d763bdadc4adf3b254f1a913949/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:46:48,082 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:46:48,084 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:46:48,084 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:46:48,085 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:46:48,085 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:46:48,086 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T12:46:48,088 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:46:48,091 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T12:46:48,091 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 9dab1d763bdadc4adf3b254f1a913949; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11744429280, jitterRate=0.09378521144390106}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T12:46:48,091 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:46:48,092 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 9dab1d763bdadc4adf3b254f1a913949: Running coprocessor pre-open hook at 1733316408071Writing region info on filesystem at 1733316408071Initializing all the Stores at 1733316408073 (+2 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316408073Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316408073Cleaning up temporary data from old regions at 1733316408085 (+12 ms)Running coprocessor post-open hooks at 1733316408091 (+6 ms)Region opened successfully at 1733316408092 (+1 ms) 2024-12-04T12:46:48,093 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., pid=28, masterSystemTime=1733316408057 2024-12-04T12:46:48,095 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:46:48,095 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:46:48,096 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9dab1d763bdadc4adf3b254f1a913949, regionState=OPEN, openSeqNum=2, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:48,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9dab1d763bdadc4adf3b254f1a913949, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:46:48,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-04T12:46:48,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 9dab1d763bdadc4adf3b254f1a913949, server=938b79ed23df,38879,1733316387992 in 198 msec 2024-12-04T12:46:48,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-04T12:46:48,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9dab1d763bdadc4adf3b254f1a913949, ASSIGN in 359 msec 2024-12-04T12:46:48,107 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T12:46:48,107 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316408107"}]},"ts":"1733316408107"} 2024-12-04T12:46:48,110 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-12-04T12:46:48,111 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T12:46:48,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system in 1.2110 sec 2024-12-04T12:46:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T12:46:49,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system completed 2024-12-04T12:46:49,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:46:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system_bulk 2024-12-04T12:46:49,087 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T12:46:49,087 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:49,087 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2024-12-04T12:46:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T12:46:49,088 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T12:46:49,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741851_1027 (size=640) 2024-12-04T12:46:49,149 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17333163965973' 2024-12-04T12:46:49,150 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-12-04T12:46:49,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T12:46:49,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T12:46:49,499 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 58f768c3c54039235f5a6360aacc8ebf, NAME => 'backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:46:49,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741852_1028 (size=45) 2024-12-04T12:46:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T12:46:49,912 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:49,913 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1722): Closing 58f768c3c54039235f5a6360aacc8ebf, disabling compactions & flushes 2024-12-04T12:46:49,913 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. 2024-12-04T12:46:49,913 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. 2024-12-04T12:46:49,914 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. after waiting 0 ms 2024-12-04T12:46:49,914 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. 2024-12-04T12:46:49,914 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1973): Closed backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. 2024-12-04T12:46:49,914 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1676): Region close journal for 58f768c3c54039235f5a6360aacc8ebf: Waiting for close lock at 1733316409913Disabling compacts and flushes for region at 1733316409913Disabling writes for close at 1733316409914 (+1 ms)Writing region close event to WAL at 1733316409914Closed at 1733316409914 2024-12-04T12:46:49,916 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T12:46:49,917 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733316409917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733316409917"}]},"ts":"1733316409917"} 2024-12-04T12:46:49,920 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T12:46:49,922 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T12:46:49,922 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316409922"}]},"ts":"1733316409922"} 2024-12-04T12:46:49,925 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2024-12-04T12:46:49,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=58f768c3c54039235f5a6360aacc8ebf, ASSIGN}] 2024-12-04T12:46:49,928 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=58f768c3c54039235f5a6360aacc8ebf, ASSIGN 2024-12-04T12:46:49,929 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=58f768c3c54039235f5a6360aacc8ebf, ASSIGN; state=OFFLINE, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=false 2024-12-04T12:46:50,080 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=58f768c3c54039235f5a6360aacc8ebf, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:50,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=58f768c3c54039235f5a6360aacc8ebf, ASSIGN because future has completed 2024-12-04T12:46:50,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58f768c3c54039235f5a6360aacc8ebf, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:46:50,188 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-12-04T12:46:50,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T12:46:50,242 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. 2024-12-04T12:46:50,242 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 58f768c3c54039235f5a6360aacc8ebf, NAME => 'backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf.', STARTKEY => '', ENDKEY => ''} 2024-12-04T12:46:50,242 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:46:50,242 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:46:50,243 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:46:50,243 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:46:50,243 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:46:50,245 INFO [StoreOpener-58f768c3c54039235f5a6360aacc8ebf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:46:50,246 INFO [StoreOpener-58f768c3c54039235f5a6360aacc8ebf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58f768c3c54039235f5a6360aacc8ebf columnFamilyName meta 2024-12-04T12:46:50,247 DEBUG [StoreOpener-58f768c3c54039235f5a6360aacc8ebf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:50,247 INFO [StoreOpener-58f768c3c54039235f5a6360aacc8ebf-1 {}] regionserver.HStore(327): Store=58f768c3c54039235f5a6360aacc8ebf/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:46:50,247 INFO [StoreOpener-58f768c3c54039235f5a6360aacc8ebf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:46:50,249 INFO [StoreOpener-58f768c3c54039235f5a6360aacc8ebf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58f768c3c54039235f5a6360aacc8ebf columnFamilyName session 2024-12-04T12:46:50,249 DEBUG [StoreOpener-58f768c3c54039235f5a6360aacc8ebf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:50,250 INFO [StoreOpener-58f768c3c54039235f5a6360aacc8ebf-1 {}] regionserver.HStore(327): Store=58f768c3c54039235f5a6360aacc8ebf/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:46:50,250 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:46:50,251 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system_bulk/58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:46:50,252 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system_bulk/58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:46:50,253 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:46:50,253 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:46:50,253 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T12:46:50,255 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:46:50,258 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system_bulk/58f768c3c54039235f5a6360aacc8ebf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T12:46:50,259 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 58f768c3c54039235f5a6360aacc8ebf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10092528000, jitterRate=-0.060060083866119385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T12:46:50,259 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:46:50,260 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 58f768c3c54039235f5a6360aacc8ebf: Running coprocessor pre-open hook at 1733316410243Writing region info on filesystem at 1733316410243Initializing all the Stores at 1733316410244 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316410244Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316410245 (+1 ms)Cleaning up temporary data from old regions at 1733316410253 (+8 ms)Running coprocessor post-open hooks at 1733316410259 (+6 ms)Region opened successfully at 1733316410260 (+1 ms) 2024-12-04T12:46:50,261 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf., pid=31, masterSystemTime=1733316410237 2024-12-04T12:46:50,263 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. 2024-12-04T12:46:50,263 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. 2024-12-04T12:46:50,264 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=58f768c3c54039235f5a6360aacc8ebf, regionState=OPEN, openSeqNum=2, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:46:50,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58f768c3c54039235f5a6360aacc8ebf, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:46:50,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-04T12:46:50,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 58f768c3c54039235f5a6360aacc8ebf, server=938b79ed23df,38879,1733316387992 in 184 msec 2024-12-04T12:46:50,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-04T12:46:50,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=58f768c3c54039235f5a6360aacc8ebf, ASSIGN in 344 msec 2024-12-04T12:46:50,274 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T12:46:50,274 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316410274"}]},"ts":"1733316410274"} 2024-12-04T12:46:50,277 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2024-12-04T12:46:50,278 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T12:46:50,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system_bulk in 1.1940 sec 2024-12-04T12:46:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T12:46:51,235 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system_bulk completed 2024-12-04T12:46:51,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:46:51,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:46:51,256 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-04T12:46:51,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='activesession:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:46:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2024-12-04T12:46:51,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T12:46:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733316411280 (current time:1733316411280). 2024-12-04T12:46:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T12:46:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-04T12:46:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59203161, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:46:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:46:51,283 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:46:51,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:46:51,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:46:51,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bca5cc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:51,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:46:51,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:46:51,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:46:51,285 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:46:51,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43fc5764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:46:51,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:46:51,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:46:51,291 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:46:51,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049. 2024-12-04T12:46:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:46:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:46:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:46:51,292 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:46:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T12:46:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T12:46:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-12-04T12:46:51,302 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T12:46:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T12:46:51,306 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T12:46:51,317 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T12:46:51,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741853_1029 (size=52) 2024-12-04T12:46:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T12:46:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T12:46:51,728 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T12:46:51,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949}] 2024-12-04T12:46:51,736 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:46:51,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38879 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-12-04T12:46:51,898 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:46:51,900 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 9dab1d763bdadc4adf3b254f1a913949 2/2 column families, dataSize=45 B heapSize=632 B 2024-12-04T12:46:51,919 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/session/0a06d46ae887482d95f1e19410656caa is 49, key is activesession:/session:c/1733316411271/Put/seqid=0 2024-12-04T12:46:51,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741854_1030 (size=5049) 2024-12-04T12:46:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T12:46:52,328 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/session/0a06d46ae887482d95f1e19410656caa 2024-12-04T12:46:52,343 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/session/0a06d46ae887482d95f1e19410656caa as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/0a06d46ae887482d95f1e19410656caa 2024-12-04T12:46:52,353 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/0a06d46ae887482d95f1e19410656caa, entries=1, sequenceid=5, filesize=4.9 K 2024-12-04T12:46:52,355 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 9dab1d763bdadc4adf3b254f1a913949 in 456ms, sequenceid=5, compaction requested=false 2024-12-04T12:46:52,355 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 9dab1d763bdadc4adf3b254f1a913949: 2024-12-04T12:46:52,355 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. for snapshot_backup_system completed. 2024-12-04T12:46:52,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949.' region-info for snapshot=snapshot_backup_system 2024-12-04T12:46:52,361 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T12:46:52,363 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-04T12:46:52,363 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/0a06d46ae887482d95f1e19410656caa] hfiles 2024-12-04T12:46:52,366 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/0a06d46ae887482d95f1e19410656caa for snapshot=snapshot_backup_system 2024-12-04T12:46:52,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741855_1031 (size=95) 2024-12-04T12:46:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T12:46:52,778 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:46:52,783 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-12-04T12:46:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-12-04T12:46:52,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:46:52,786 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:46:52,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-04T12:46:52,790 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T12:46:52,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949 in 1.0550 sec 2024-12-04T12:46:52,793 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T12:46:52,794 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T12:46:52,794 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-04T12:46:52,796 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-04T12:46:52,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741856_1032 (size=731) 2024-12-04T12:46:53,218 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T12:46:53,231 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T12:46:53,232 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_backup_system 2024-12-04T12:46:53,234 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T12:46:53,234 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-12-04T12:46:53,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.9370 sec 2024-12-04T12:46:53,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T12:46:53,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-04T12:46:53,459 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733316406744 started at 1733316413456. 2024-12-04T12:46:53,477 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733316406744 has been started. 2024-12-04T12:46:53,481 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-12-04T12:46:53,486 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-12-04T12:46:53,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-04T12:46:53,489 INFO [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-04T12:46:53,489 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-04T12:46:53,489 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-04T12:46:53,489 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-04T12:46:53,490 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T12:46:53,490 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T12:46:53,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T12:46:53,559 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,560 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T12:46:53,560 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T12:46:53,561 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,561 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-04T12:46:53,561 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-04T12:46:53,562 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T12:46:53,563 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-04T12:46:53,564 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T12:46:53,564 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-04T12:46:53,566 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-04T12:46:53,566 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-04T12:46:53,567 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-04T12:46:53,567 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-04T12:46:53,568 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-04T12:46:53,568 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-04T12:46:53,568 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '938b79ed23df,38879,1733316387992' joining acquired barrier for procedure (rolllog) in zk 2024-12-04T12:46:53,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,575 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-04T12:46:53,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T12:46:53,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T12:46:53,576 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-04T12:46:53,576 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-04T12:46:53,577 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T12:46:53,577 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T12:46:53,578 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:46:53,578 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,579 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T12:46:53,579 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '938b79ed23df,38879,1733316387992' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-04T12:46:53,579 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@59bcff7[Count = 0] remaining members to acquire global barrier 2024-12-04T12:46:53,579 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-04T12:46:53,579 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-04T12:46:53,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T12:46:53,584 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-04T12:46:53,584 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-04T12:46:53,584 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,584 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-04T12:46:53,584 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-04T12:46:53,585 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-04T12:46:53,585 DEBUG [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,591 INFO [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733316391322 highest: 1733316398288 on 938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,592 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0:(num 1733316391322) roll requested 2024-12-04T12:46:53,609 DEBUG [regionserver/938b79ed23df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316413592, exclude list is [], retry=0 2024-12-04T12:46:53,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:46:53,616 INFO [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316391322 with entries=3, filesize=1.00 KB; new WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316413592 2024-12-04T12:46:53,617 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:46:53,617 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316391322 is not closed yet, will try archiving it next time 2024-12-04T12:46:53,617 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 938b79ed23df%2C38879%2C1733316387992.meta:.meta(num 1733316392177) roll requested 2024-12-04T12:46:53,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741833_1009 (size=1034) 2024-12-04T12:46:53,621 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316391322 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316391322 2024-12-04T12:46:53,636 DEBUG [regionserver/938b79ed23df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316413617.meta, exclude list is [], retry=0 2024-12-04T12:46:53,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:46:53,641 INFO [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316392177.meta with entries=42, filesize=11.76 KB; new WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316413617.meta 2024-12-04T12:46:53,641 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:46:53,641 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316392177.meta is not closed yet, will try archiving it next time 2024-12-04T12:46:53,641 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1:(num 1733316398288) roll requested 2024-12-04T12:46:53,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741834_1010 (size=12055) 2024-12-04T12:46:53,643 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316392177.meta not finished, retry = 0 2024-12-04T12:46:53,653 DEBUG [regionserver/938b79ed23df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316413641, exclude list is [], retry=0 2024-12-04T12:46:53,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:46:53,658 INFO [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316398288 with entries=24, filesize=4.64 KB; new WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316413641 2024-12-04T12:46:53,658 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:46:53,658 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316398288 is not closed yet, will try archiving it next time 2024-12-04T12:46:53,660 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316398288 not finished, retry = 0 2024-12-04T12:46:53,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741840_1016 (size=4756) 2024-12-04T12:46:53,672 DEBUG [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 81 2024-12-04T12:46:53,673 INFO [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733316413592 on 938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,673 DEBUG [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:46:53,673 DEBUG [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:46:53,673 DEBUG [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7dfee8 2024-12-04T12:46:53,673 DEBUG [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:46:53,674 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44721, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-04T12:46:53,675 DEBUG [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:46:53,675 DEBUG [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:46:53,676 DEBUG [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:46:53,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56039, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-04T12:46:53,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:46:53,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:46:53,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='rslogts:hdfs://localhost:41555/backupUT\x00', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:46:53,698 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-04T12:46:53,699 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '938b79ed23df,38879,1733316387992' in zk 2024-12-04T12:46:53,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,709 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-04T12:46:53,709 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T12:46:53,709 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,709 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-04T12:46:53,709 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T12:46:53,709 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T12:46:53,710 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T12:46:53,710 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T12:46:53,710 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:46:53,711 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,711 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T12:46:53,712 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:46:53,712 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,713 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '938b79ed23df,38879,1733316387992': 2024-12-04T12:46:53,713 INFO [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-04T12:46:53,713 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-04T12:46:53,713 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-04T12:46:53,713 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-04T12:46:53,713 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '938b79ed23df,38879,1733316387992' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-04T12:46:53,713 INFO [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-04T12:46:53,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T12:46:53,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T12:46:53,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T12:46:53,717 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-04T12:46:53,717 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-04T12:46:53,717 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T12:46:53,717 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T12:46:53,717 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T12:46:53,718 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T12:46:53,718 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T12:46:53,718 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T12:46:53,718 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,718 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:46:53,718 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T12:46:53,718 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T12:46:53,719 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:46:53,719 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,719 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T12:46:53,720 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,720 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:46:53,720 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T12:46:53,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T12:46:53,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T12:46:53,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T12:46:53,734 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T12:46:53,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:46:53,734 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T12:46:53,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T12:46:53,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T12:46:53,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T12:46:53,734 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T12:46:53,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-04T12:46:53,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-04T12:46:53,735 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T12:46:53,735 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T12:46:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-04T12:46:53,751 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733316413750_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } 2024-12-04T12:46:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733316413751 (current time:1733316413751). 2024-12-04T12:46:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T12:46:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733316413750_default_test-1733316396597 VERSION not specified, setting to 2 2024-12-04T12:46:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cdd666a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:46:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:46:53,752 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:46:53,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:46:53,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:46:53,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5655a5ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:53,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:46:53,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:46:53,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:46:53,754 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:46:53,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@377ba06f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:46:53,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:46:53,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:46:53,757 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:46:53,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049. 2024-12-04T12:46:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:46:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:46:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:46:53,759 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:46:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T12:46:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733316413750_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } 2024-12-04T12:46:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733316413750_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-12-04T12:46:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T12:46:53,763 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733316413750_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T12:46:53,764 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733316413750_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T12:46:53,768 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733316413750_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T12:46:53,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741860_1036 (size=84) 2024-12-04T12:46:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T12:46:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T12:46:54,180 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733316413750_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T12:46:54,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 87e3e36a673c628af1440164c1fd3443}] 2024-12-04T12:46:54,183 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:54,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-12-04T12:46:54,338 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:54,338 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 87e3e36a673c628af1440164c1fd3443 3/3 column families, dataSize=654 B heapSize=2.25 KB 2024-12-04T12:46:54,362 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/f3/027afb5687a04f3cb770de8c58f62ade is 58, key is row-test-1733316396597-3-0/f3:q1/1733316406715/Put/seqid=0 2024-12-04T12:46:54,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741861_1037 (size=5387) 2024-12-04T12:46:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T12:46:54,769 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/f3/027afb5687a04f3cb770de8c58f62ade 2024-12-04T12:46:54,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741862_1038 (size=4469) 2024-12-04T12:46:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T12:46:55,152 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2024-12-04T12:46:55,219 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/mob/6759befaa3d44e67a2be163409a2ab50 is 59, key is row-test-1733316396597-3-0/mob:q1/1733316406729/Put/seqid=0 2024-12-04T12:46:55,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741863_1039 (size=5484) 2024-12-04T12:46:55,582 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T12:46:55,631 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=330, hasBloomFilter=true, into tmp file hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/mob/6759befaa3d44e67a2be163409a2ab50 2024-12-04T12:46:55,643 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/f3/027afb5687a04f3cb770de8c58f62ade as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f3/027afb5687a04f3cb770de8c58f62ade 2024-12-04T12:46:55,651 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f3/027afb5687a04f3cb770de8c58f62ade, entries=6, sequenceid=121, filesize=5.3 K 2024-12-04T12:46:55,653 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/mob/6759befaa3d44e67a2be163409a2ab50 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/6759befaa3d44e67a2be163409a2ab50 2024-12-04T12:46:55,662 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/6759befaa3d44e67a2be163409a2ab50, entries=6, sequenceid=121, filesize=5.4 K 2024-12-04T12:46:55,663 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~654 B/654, heapSize ~1.97 KB/2016, currentSize=0 B/0 for 87e3e36a673c628af1440164c1fd3443 in 1325ms, sequenceid=121, compaction requested=false 2024-12-04T12:46:55,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 87e3e36a673c628af1440164c1fd3443: 2024-12-04T12:46:55,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. for snapshot_1733316413750_default_test-1733316396597 completed. 2024-12-04T12:46:55,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443.' region-info for snapshot=snapshot_1733316413750_default_test-1733316396597 2024-12-04T12:46:55,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T12:46:55,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40] hfiles 2024-12-04T12:46:55,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40 for snapshot=snapshot_1733316413750_default_test-1733316396597 2024-12-04T12:46:55,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f3/027afb5687a04f3cb770de8c58f62ade] hfiles 2024-12-04T12:46:55,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f3/027afb5687a04f3cb770de8c58f62ade for snapshot=snapshot_1733316413750_default_test-1733316396597 2024-12-04T12:46:55,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/6759befaa3d44e67a2be163409a2ab50] hfiles 2024-12-04T12:46:55,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/6759befaa3d44e67a2be163409a2ab50 for snapshot=snapshot_1733316413750_default_test-1733316396597 2024-12-04T12:46:55,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741864_1040 (size=185) 2024-12-04T12:46:55,745 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-12-04T12:46:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T12:46:56,072 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:46:56,072 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-12-04T12:46:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-12-04T12:46:56,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733316413750_default_test-1733316396597 on region 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:56,073 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:46:56,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-04T12:46:56,079 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733316413750_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T12:46:56,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 87e3e36a673c628af1440164c1fd3443 in 1.8950 sec 2024-12-04T12:46:56,081 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733316413750_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T12:46:56,085 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-12-04T12:46:56,085 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-12-04T12:46:56,086 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:56,087 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-12-04T12:46:56,087 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:56,088 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f3 2024-12-04T12:46:56,088 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:46:56,088 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: mob 2024-12-04T12:46:56,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741865_1041 (size=49) 2024-12-04T12:46:56,524 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733316413750_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T12:46:56,525 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733316413750_default_test-1733316396597 2024-12-04T12:46:56,526 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_1733316413750_default_test-1733316396597 2024-12-04T12:46:56,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741866_1042 (size=1146) 2024-12-04T12:46:56,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-12-04T12:46:56,830 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2024-12-04T12:46:56,832 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-12-04T12:46:56,832 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2024-12-04T12:46:56,950 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733316413750_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T12:46:56,963 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733316413750_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T12:46:56,964 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_1733316413750_default_test-1733316396597 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_1733316413750_default_test-1733316396597 2024-12-04T12:46:56,966 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733316413750_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T12:46:56,966 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733316413750_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-12-04T12:46:56,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733316413750_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } in 3.2060 sec 2024-12-04T12:46:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T12:46:57,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1733316396597 completed 2024-12-04T12:46:57,948 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733316417946_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } 2024-12-04T12:46:57,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733316417948 (current time:1733316417948). 2024-12-04T12:46:57,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T12:46:57,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733316417946_ns2_test-17333163965971 VERSION not specified, setting to 2 2024-12-04T12:46:57,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@147c94c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:57,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:46:57,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:46:57,953 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:46:57,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:46:57,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:46:57,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388debb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:57,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:46:57,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:46:57,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:46:57,956 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:46:57,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7346cc14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:46:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:46:57,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:46:57,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:46:57,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:46:57,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049. 2024-12-04T12:46:57,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:46:57,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:46:57,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:46:57,961 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:46:57,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T12:46:57,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733316417946_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } 2024-12-04T12:46:57,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733316417946_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-12-04T12:46:57,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T12:46:57,964 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733316417946_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T12:46:57,965 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733316417946_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T12:46:57,968 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733316417946_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T12:46:57,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741867_1043 (size=86) 2024-12-04T12:46:58,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T12:46:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T12:46:58,379 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733316417946_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T12:46:58,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c988c84788a01c971b5789026625491a}] 2024-12-04T12:46:58,384 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c988c84788a01c971b5789026625491a 2024-12-04T12:46:58,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2024-12-04T12:46:58,540 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:46:58,541 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing c988c84788a01c971b5789026625491a 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-12-04T12:46:58,564 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/.tmp/f/194e38d3f9c54ed88dcc2848906ca7b4 is 37, key is row10/f:q1/1733316401612/Put/seqid=0 2024-12-04T12:46:58,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741868_1044 (size=8413) 2024-12-04T12:46:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T12:46:58,970 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/.tmp/f/194e38d3f9c54ed88dcc2848906ca7b4 2024-12-04T12:46:58,984 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/.tmp/f/194e38d3f9c54ed88dcc2848906ca7b4 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/194e38d3f9c54ed88dcc2848906ca7b4 2024-12-04T12:46:58,991 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/194e38d3f9c54ed88dcc2848906ca7b4, entries=99, sequenceid=103, filesize=8.2 K 2024-12-04T12:46:58,992 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for c988c84788a01c971b5789026625491a in 452ms, sequenceid=103, compaction requested=false 2024-12-04T12:46:58,992 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for c988c84788a01c971b5789026625491a: 2024-12-04T12:46:58,992 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. for snapshot_1733316417946_ns2_test-17333163965971 completed. 2024-12-04T12:46:58,992 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a.' region-info for snapshot=snapshot_1733316417946_ns2_test-17333163965971 2024-12-04T12:46:58,993 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T12:46:58,993 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/194e38d3f9c54ed88dcc2848906ca7b4] hfiles 2024-12-04T12:46:58,993 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/194e38d3f9c54ed88dcc2848906ca7b4 for snapshot=snapshot_1733316417946_ns2_test-17333163965971 2024-12-04T12:46:58,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741869_1045 (size=91) 2024-12-04T12:46:59,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T12:46:59,402 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:46:59,402 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-04T12:46:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-04T12:46:59,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733316417946_ns2_test-17333163965971 on region c988c84788a01c971b5789026625491a 2024-12-04T12:46:59,405 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c988c84788a01c971b5789026625491a 2024-12-04T12:46:59,413 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-04T12:46:59,413 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733316417946_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T12:46:59,413 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; SnapshotRegionProcedure c988c84788a01c971b5789026625491a in 1.0280 sec 2024-12-04T12:46:59,414 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733316417946_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T12:46:59,415 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733316417946_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T12:46:59,416 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733316417946_ns2_test-17333163965971 2024-12-04T12:46:59,417 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_1733316417946_ns2_test-17333163965971 2024-12-04T12:46:59,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741870_1046 (size=456) 2024-12-04T12:46:59,837 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733316417946_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T12:46:59,846 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733316417946_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T12:46:59,846 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_1733316417946_ns2_test-17333163965971 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_1733316417946_ns2_test-17333163965971 2024-12-04T12:46:59,848 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733316417946_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T12:46:59,848 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733316417946_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-12-04T12:46:59,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733316417946_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } in 1.8860 sec 2024-12-04T12:47:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T12:47:00,115 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17333163965971 completed 2024-12-04T12:47:00,115 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1733316406744 2024-12-04T12:47:00,115 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-12-04T12:47:00,118 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2024-12-04T12:47:00,123 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733316406744_test-1733316396597 2024-12-04T12:47:00,123 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733316413750_default_test-1733316396597 to hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/ 2024-12-04T12:47:00,129 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-04T12:47:00,157 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:41555, inputRoot=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:47:00,157 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_430596070_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/.hbase-snapshot/.tmp/snapshot_1733316413750_default_test-1733316396597 2024-12-04T12:47:00,159 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-04T12:47:00,166 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_1733316413750_default_test-1733316396597 to hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/.hbase-snapshot/.tmp/snapshot_1733316413750_default_test-1733316396597 2024-12-04T12:47:00,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741872_1048 (size=84) 2024-12-04T12:47:00,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741871_1047 (size=1146) 2024-12-04T12:47:00,593 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:00,593 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:00,593 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:01,378 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-12152082871746603271.jar 2024-12-04T12:47:01,379 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:01,379 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:01,380 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:01,380 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:01,380 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:01,381 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:01,381 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:01,381 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:01,382 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:01,382 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:47:01,382 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T12:47:01,382 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T12:47:01,383 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T12:47:01,383 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T12:47:01,383 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T12:47:01,384 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T12:47:01,384 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T12:47:01,384 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T12:47:01,384 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T12:47:01,385 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T12:47:01,387 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:01,387 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:01,387 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:01,387 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:01,388 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:01,388 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:01,388 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:01,445 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:47:01,473 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733316413750_default_test-1733316396597' hfile list 2024-12-04T12:47:01,480 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=18.8 K 2024-12-04T12:47:01,689 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local1709972678_0001_2191a381-a1dc-40f6-badf-41e2fcf2b17b/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T12:47:01,689 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local1709972678_0001_2191a381-a1dc-40f6-badf-41e2fcf2b17b/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T12:47:01,838 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-04T12:47:01,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741873_1049 (size=8413) 2024-12-04T12:47:02,316 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40 output=hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40 2024-12-04T12:47:02,318 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.017M/sec 2024-12-04T12:47:02,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741874_1050 (size=5484) 2024-12-04T12:47:02,756 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/6759befaa3d44e67a2be163409a2ab50 output=hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/6759befaa3d44e67a2be163409a2ab50 2024-12-04T12:47:02,756 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-12-04T12:47:02,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741875_1051 (size=5387) 2024-12-04T12:47:03,214 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f3/027afb5687a04f3cb770de8c58f62ade output=hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f3/027afb5687a04f3cb770de8c58f62ade 2024-12-04T12:47:03,214 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5387 (5.3 K) time=0sec 0.011M/sec 2024-12-04T12:47:03,761 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-04T12:47:03,763 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-04T12:47:03,774 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733316413750_default_test-1733316396597 2024-12-04T12:47:03,774 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733316413750_default_test-1733316396597 finished. 2024-12-04T12:47:03,774 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733316406744_ns2:test-17333163965971 2024-12-04T12:47:03,774 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733316417946_ns2_test-17333163965971 to hdfs://localhost:41555/backupUT/backup_1733316406744/ns2/test-17333163965971/ 2024-12-04T12:47:03,774 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-04T12:47:03,803 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:41555, inputRoot=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:47:03,803 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_430596070_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:41555/backupUT/backup_1733316406744/ns2/test-17333163965971, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:41555/backupUT/backup_1733316406744/ns2/test-17333163965971/.hbase-snapshot/.tmp/snapshot_1733316417946_ns2_test-17333163965971 2024-12-04T12:47:03,805 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-04T12:47:03,810 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_1733316417946_ns2_test-17333163965971 to hdfs://localhost:41555/backupUT/backup_1733316406744/ns2/test-17333163965971/.hbase-snapshot/.tmp/snapshot_1733316417946_ns2_test-17333163965971 2024-12-04T12:47:03,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741877_1053 (size=456) 2024-12-04T12:47:03,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741876_1052 (size=86) 2024-12-04T12:47:04,233 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:04,234 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:04,235 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:05,005 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-15114951350967064483.jar 2024-12-04T12:47:05,006 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:05,006 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:05,006 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:05,007 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:05,007 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:05,007 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:05,008 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:05,008 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:05,008 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:05,008 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:47:05,008 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T12:47:05,009 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T12:47:05,009 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T12:47:05,009 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T12:47:05,009 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T12:47:05,010 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T12:47:05,010 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T12:47:05,010 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T12:47:05,010 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T12:47:05,011 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T12:47:05,011 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:05,011 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:05,011 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:05,012 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:05,012 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:05,012 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:05,012 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:05,048 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:47:05,070 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733316417946_ns2_test-17333163965971' hfile list 2024-12-04T12:47:05,073 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=8.2 K 2024-12-04T12:47:05,152 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local866079248_0002_9e4ae428-bae2-4d2d-827e-7214b3e518dd/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T12:47:05,152 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local866079248_0002_9e4ae428-bae2-4d2d-827e-7214b3e518dd/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T12:47:05,221 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-04T12:47:05,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741878_1054 (size=8413) 2024-12-04T12:47:05,672 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/194e38d3f9c54ed88dcc2848906ca7b4 output=hdfs://localhost:41555/backupUT/backup_1733316406744/ns2/test-17333163965971/archive/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/194e38d3f9c54ed88dcc2848906ca7b4 2024-12-04T12:47:05,673 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-12-04T12:47:06,194 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-04T12:47:06,196 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-04T12:47:06,202 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733316417946_ns2_test-17333163965971 2024-12-04T12:47:06,202 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733316417946_ns2_test-17333163965971 finished. 2024-12-04T12:47:06,203 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1733316396597 2024-12-04T12:47:06,203 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17333163965971 2024-12-04T12:47:06,248 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733316406744 2024-12-04T12:47:06,248 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-12-04T12:47:06,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741879_1055 (size=97) 2024-12-04T12:47:06,658 INFO [Time-limited test {}] impl.BackupManifest(504): Manifest file stored to hdfs://localhost:41555/backupUT/backup_1733316406744/.backup.manifest 2024-12-04T12:47:06,663 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1733316406744 finished: type=FULL,tablelist=test-1733316396597;ns2:test-17333163965971,targetRootDir=hdfs://localhost:41555/backupUT,startts=1733316413456,completets=1733316426246,bytescopied=0 2024-12-04T12:47:06,663 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-12-04T12:47:06,664 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733316413750_default_test-1733316396597 2024-12-04T12:47:06,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733316413750_default_test-1733316396597" type: DISABLED 2024-12-04T12:47:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733316413750_default_test-1733316396597 2024-12-04T12:47:06,673 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733316413750_default_test-1733316396597 for backup backup_1733316406744 succeeded. 2024-12-04T12:47:06,673 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733316417946_ns2_test-17333163965971 2024-12-04T12:47:06,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733316417946_ns2_test-17333163965971" type: DISABLED 2024-12-04T12:47:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733316417946_ns2_test-17333163965971 2024-12-04T12:47:06,678 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733316417946_ns2_test-17333163965971 for backup backup_1733316406744 succeeded. 2024-12-04T12:47:06,679 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-04T12:47:06,689 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-12-04T12:47:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-04T12:47:06,693 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-04T12:47:06,697 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-04T12:47:06,712 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1733316406744 completed. 2024-12-04T12:47:06,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:backup_1733316406744', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:47:06,739 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:41555/backupUT/backup_1733316406744/.backup.manifest 2024-12-04T12:47:06,742 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733316406744/.backup.manifest 2024-12-04T12:47:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:06,951 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(116): writing 99 rows to test-1733316396597 2024-12-04T12:47:06,972 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(119): written 99 rows to test-1733316396597 2024-12-04T12:47:07,149 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(123): written 111 rows to test-1733316396597 to Mob enabled CF 2024-12-04T12:47:07,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17333163965971', row='row-t20', locateType=CURRENT is [region=ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:47:07,179 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(136): written 5 rows to ns2:test-17333163965971 2024-12-04T12:47:07,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$3(2313): Client=jenkins//172.17.0.3 split test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:47:07,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] assignment.SplitTableRegionProcedure(210): test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. splitKey isn't explicitly specified, will try to find a best split key from RS 938b79ed23df,38879,1733316387992 2024-12-04T12:47:07,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40 because midkey is the same as first or last row 2024-12-04T12:47:07,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f3/027afb5687a04f3cb770de8c58f62ade because midkey is the same as first or last row 2024-12-04T12:47:07,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/6759befaa3d44e67a2be163409a2ab50 because midkey is the same as first or last row 2024-12-04T12:47:07,190 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.HRegion(2902): Flushing 87e3e36a673c628af1440164c1fd3443 3/3 column families, dataSize=11.48 KB heapSize=27.79 KB 2024-12-04T12:47:07,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/f/ed2773986acc4ae18188f34dec668da1 is 59, key is row-test-1733316396597-1-10/f:q1/1733316426782/Put/seqid=0 2024-12-04T12:47:07,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741880_1056 (size=10796) 2024-12-04T12:47:07,616 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/f/ed2773986acc4ae18188f34dec668da1 2024-12-04T12:47:07,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f4e86055bf4b405ab07a7b9c2fae1c0b_87e3e36a673c628af1440164c1fd3443 is 63, key is row-test-1733316396597-3-100/mob:q1/1733316427129/Put/seqid=0 2024-12-04T12:47:07,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741881_1057 (size=5591) 2024-12-04T12:47:08,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:47:08,053 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f4e86055bf4b405ab07a7b9c2fae1c0b_87e3e36a673c628af1440164c1fd3443 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/mobdir/data/default/test-1733316396597/64682aa8ca2d45d6ee9ace61337cabc4/mob/d41d8cd98f00b204e9800998ecf8427e20241204f4e86055bf4b405ab07a7b9c2fae1c0b_87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:08,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/mob/5799223a4841435fb10d4958b6b73deb, store: [table=test-1733316396597 family=mob region=87e3e36a673c628af1440164c1fd3443] 2024-12-04T12:47:08,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/mob/5799223a4841435fb10d4958b6b73deb is 192, key is row-test-1733316396597-3-100/mob:q1/1733316427129/Put/seqid=0 2024-12-04T12:47:08,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741882_1058 (size=13760) 2024-12-04T12:47:08,462 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=6.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/mob/5799223a4841435fb10d4958b6b73deb 2024-12-04T12:47:08,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/f/ed2773986acc4ae18188f34dec668da1 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/ed2773986acc4ae18188f34dec668da1 2024-12-04T12:47:08,481 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/ed2773986acc4ae18188f34dec668da1, entries=99, sequenceid=334, filesize=10.5 K 2024-12-04T12:47:08,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/mob/5799223a4841435fb10d4958b6b73deb as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/5799223a4841435fb10d4958b6b73deb 2024-12-04T12:47:08,488 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/5799223a4841435fb10d4958b6b73deb, entries=111, sequenceid=334, filesize=13.4 K 2024-12-04T12:47:08,490 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.48 KB/11754, heapSize ~27.51 KB/28168, currentSize=0 B/0 for 87e3e36a673c628af1440164c1fd3443 in 1299ms, sequenceid=334, compaction requested=false 2024-12-04T12:47:08,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.HRegion(2603): Flush status journal for 87e3e36a673c628af1440164c1fd3443: 2024-12-04T12:47:08,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/ed2773986acc4ae18188f34dec668da1 because midkey is the same as first or last row 2024-12-04T12:47:08,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f3/027afb5687a04f3cb770de8c58f62ade because midkey is the same as first or last row 2024-12-04T12:47:08,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/5799223a4841435fb10d4958b6b73deb because midkey is the same as first or last row 2024-12-04T12:47:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=938b79ed23df,38879,1733316387992 2024-12-04T12:47:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-04T12:47:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.CallRunner(138): callId: 303 service: MasterService methodName: SplitRegion size: 100 connection: 172.17.0.3:40306 deadline: 1733316487185, exception=org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row 2024-12-04T12:47:08,496 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(147): region is not splittable, because java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:47:08,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: SPLIT_REGION, Table Name: default:test-1733316396597 failed with org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:47:08,497 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T12:47:08,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733316396597,, stopping at row=test-1733316396597 ,, for max=2147483647 with caching=100 2024-12-04T12:47:08,507 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733316396597,, stopping at row=test-1733316396597 ,, for max=2147483647 with caching=100 2024-12-04T12:47:08,515 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(152): split finished in =1336 2024-12-04T12:47:08,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:08,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:08,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:08,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:08,558 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-04T12:47:08,562 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733316428515 2024-12-04T12:47:08,569 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733316406744} 2024-12-04T12:47:08,569 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-12-04T12:47:08,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:08,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:08,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:08,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:08,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T12:47:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733316428620 (current time:1733316428620). 2024-12-04T12:47:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T12:47:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-04T12:47:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65f5b1c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:47:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:47:08,622 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:47:08,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:47:08,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:47:08,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b625aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:08,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:47:08,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:47:08,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:08,625 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:47:08,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1360a865, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:08,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:47:08,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:47:08,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:47:08,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:47:08,629 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049. 2024-12-04T12:47:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:47:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:08,629 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:47:08,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T12:47:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T12:47:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-12-04T12:47:08,632 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T12:47:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T12:47:08,633 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T12:47:08,636 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T12:47:08,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741883_1059 (size=52) 2024-12-04T12:47:08,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T12:47:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T12:47:09,044 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T12:47:09,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949}] 2024-12-04T12:47:09,046 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:47:09,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38879 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2024-12-04T12:47:09,200 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:47:09,200 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2902): Flushing 9dab1d763bdadc4adf3b254f1a913949 2/2 column families, dataSize=1.44 KB heapSize=2.75 KB 2024-12-04T12:47:09,225 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/meta/6bb6c66690d84dfb8bc5bd7b77d4a104 is 158, key is trslm:hdfs://localhost:41555/backupUT\x00ns2:test-17333163965971/meta:log-roll-map/1733316426234/Put/seqid=0 2024-12-04T12:47:09,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741884_1060 (size=5914) 2024-12-04T12:47:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T12:47:09,631 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/meta/6bb6c66690d84dfb8bc5bd7b77d4a104 2024-12-04T12:47:09,647 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/session/5d1327784fd34e43a68805beada718ac is 433, key is session:backup_1733316406744/session:context/1733316426696/Put/seqid=0 2024-12-04T12:47:09,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741885_1061 (size=5535) 2024-12-04T12:47:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T12:47:10,054 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/session/5d1327784fd34e43a68805beada718ac 2024-12-04T12:47:10,068 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/meta/6bb6c66690d84dfb8bc5bd7b77d4a104 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/6bb6c66690d84dfb8bc5bd7b77d4a104 2024-12-04T12:47:10,075 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/6bb6c66690d84dfb8bc5bd7b77d4a104, entries=6, sequenceid=17, filesize=5.8 K 2024-12-04T12:47:10,076 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/session/5d1327784fd34e43a68805beada718ac as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/5d1327784fd34e43a68805beada718ac 2024-12-04T12:47:10,083 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/5d1327784fd34e43a68805beada718ac, entries=2, sequenceid=17, filesize=5.4 K 2024-12-04T12:47:10,084 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3140): Finished flush of dataSize ~1.44 KB/1478, heapSize ~2.72 KB/2784, currentSize=0 B/0 for 9dab1d763bdadc4adf3b254f1a913949 in 884ms, sequenceid=17, compaction requested=false 2024-12-04T12:47:10,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2603): Flush status journal for 9dab1d763bdadc4adf3b254f1a913949: 2024-12-04T12:47:10,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. for snapshot_backup_system completed. 2024-12-04T12:47:10,085 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949.' region-info for snapshot=snapshot_backup_system 2024-12-04T12:47:10,085 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T12:47:10,085 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/6bb6c66690d84dfb8bc5bd7b77d4a104] hfiles 2024-12-04T12:47:10,085 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/6bb6c66690d84dfb8bc5bd7b77d4a104 for snapshot=snapshot_backup_system 2024-12-04T12:47:10,085 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/0a06d46ae887482d95f1e19410656caa, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/5d1327784fd34e43a68805beada718ac] hfiles 2024-12-04T12:47:10,085 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/0a06d46ae887482d95f1e19410656caa for snapshot=snapshot_backup_system 2024-12-04T12:47:10,085 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/5d1327784fd34e43a68805beada718ac for snapshot=snapshot_backup_system 2024-12-04T12:47:10,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741886_1062 (size=173) 2024-12-04T12:47:10,493 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:47:10,493 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-12-04T12:47:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-12-04T12:47:10,495 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:47:10,495 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:47:10,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-04T12:47:10,503 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T12:47:10,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949 in 1.4540 sec 2024-12-04T12:47:10,504 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T12:47:10,504 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T12:47:10,504 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-04T12:47:10,505 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-04T12:47:10,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741887_1063 (size=810) 2024-12-04T12:47:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T12:47:10,922 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T12:47:10,933 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T12:47:10,933 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_backup_system 2024-12-04T12:47:10,935 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T12:47:10,935 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-12-04T12:47:10,937 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.3050 sec 2024-12-04T12:47:12,480 DEBUG [master/938b79ed23df:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733316396597 2024-12-04T12:47:12,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741888_1064 (size=950) 2024-12-04T12:47:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T12:47:12,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-04T12:47:12,795 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733316428515 started at 1733316432795. 2024-12-04T12:47:12,797 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733316428515 has been started. 2024-12-04T12:47:12,806 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [ns2:test-17333163965971, test-1733316396597] 2024-12-04T12:47:12,810 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1733316398288for backupID backup_1733316428515 2024-12-04T12:47:12,810 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-12-04T12:47:12,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-12-04T12:47:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-04T12:47:12,811 INFO [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-04T12:47:12,811 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-04T12:47:12,811 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-04T12:47:12,811 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-04T12:47:12,812 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T12:47:12,812 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T12:47:12,893 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/.tabledesc/.tableinfo.0000000002.908 2024-12-04T12:47:12,893 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/.tabledesc/.tableinfo.0000000003.950 2024-12-04T12:47:12,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733316396597 in 417 msec 2024-12-04T12:47:12,907 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:12,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T12:47:12,907 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T12:47:12,908 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T12:47:12,908 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:12,908 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-04T12:47:12,908 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-04T12:47:12,909 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T12:47:12,910 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-04T12:47:12,910 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T12:47:12,910 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-04T12:47:12,911 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-04T12:47:12,911 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-04T12:47:12,911 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-04T12:47:12,911 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-04T12:47:12,911 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-04T12:47:12,911 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-04T12:47:12,911 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '938b79ed23df,38879,1733316387992' joining acquired barrier for procedure (rolllog) in zk 2024-12-04T12:47:12,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:12,916 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-04T12:47:12,916 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:12,916 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T12:47:12,916 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T12:47:12,917 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-04T12:47:12,917 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-04T12:47:12,917 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T12:47:12,918 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T12:47:12,918 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:47:12,919 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:47:12,920 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T12:47:12,920 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '938b79ed23df,38879,1733316387992' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-04T12:47:12,920 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@374dce91[Count = 0] remaining members to acquire global barrier 2024-12-04T12:47:12,920 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-04T12:47:12,920 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-04T12:47:12,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T12:47:12,925 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-04T12:47:12,925 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-04T12:47:12,925 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-04T12:47:12,925 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:12,925 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-04T12:47:12,925 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-04T12:47:12,925 DEBUG [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 938b79ed23df,38879,1733316387992 2024-12-04T12:47:12,926 INFO [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733316413592 highest: 1733316413641 on 938b79ed23df,38879,1733316387992 2024-12-04T12:47:12,926 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0:(num 1733316413592) roll requested 2024-12-04T12:47:12,937 DEBUG [regionserver/938b79ed23df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316432926, exclude list is [], retry=0 2024-12-04T12:47:12,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:47:12,942 INFO [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316413592 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316432926 2024-12-04T12:47:12,942 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:47:12,942 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316413592 is not closed yet, will try archiving it next time 2024-12-04T12:47:12,942 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 938b79ed23df%2C38879%2C1733316387992.meta:.meta(num 1733316413617) roll requested 2024-12-04T12:47:12,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741857_1033 (size=1198) 2024-12-04T12:47:12,944 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316413592 not finished, retry = 0 2024-12-04T12:47:12,954 DEBUG [regionserver/938b79ed23df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316432943.meta, exclude list is [], retry=0 2024-12-04T12:47:12,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:47:12,958 INFO [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316413617.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316432943.meta 2024-12-04T12:47:12,958 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:47:12,958 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316413617.meta is not closed yet, will try archiving it next time 2024-12-04T12:47:12,958 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1:(num 1733316413641) roll requested 2024-12-04T12:47:12,960 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316413617.meta not finished, retry = 0 2024-12-04T12:47:12,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741858_1034 (size=93) 2024-12-04T12:47:12,971 DEBUG [regionserver/938b79ed23df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316432958, exclude list is [], retry=0 2024-12-04T12:47:12,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:47:12,976 INFO [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316413641 with entries=224, filesize=29.77 KB; new WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316432958 2024-12-04T12:47:12,976 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:47:12,976 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316413641 is not closed yet, will try archiving it next time 2024-12-04T12:47:12,977 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316398288 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316398288 2024-12-04T12:47:12,977 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316413641 not finished, retry = 0 2024-12-04T12:47:12,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741859_1035 (size=30490) 2024-12-04T12:47:12,986 DEBUG [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-12-04T12:47:12,986 INFO [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733316432926 on 938b79ed23df,38879,1733316387992 2024-12-04T12:47:12,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:13,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:13,009 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-04T12:47:13,010 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '938b79ed23df,38879,1733316387992' in zk 2024-12-04T12:47:13,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:13,016 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-04T12:47:13,016 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T12:47:13,016 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-04T12:47:13,016 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:13,016 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T12:47:13,016 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T12:47:13,017 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T12:47:13,017 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T12:47:13,018 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:47:13,018 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:47:13,019 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T12:47:13,019 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:47:13,020 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:47:13,020 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '938b79ed23df,38879,1733316387992': 2024-12-04T12:47:13,021 INFO [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-04T12:47:13,021 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-04T12:47:13,021 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '938b79ed23df,38879,1733316387992' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-04T12:47:13,021 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-04T12:47:13,021 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-04T12:47:13,021 INFO [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-04T12:47:13,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T12:47:13,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T12:47:13,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T12:47:13,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-04T12:47:13,024 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-04T12:47:13,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T12:47:13,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T12:47:13,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T12:47:13,025 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T12:47:13,025 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T12:47:13,025 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T12:47:13,025 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:13,025 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T12:47:13,025 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:47:13,026 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T12:47:13,026 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:47:13,026 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:47:13,026 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:13,027 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T12:47:13,027 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:47:13,027 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:47:13,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:13,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T12:47:13,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T12:47:13,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T12:47:13,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:13,033 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T12:47:13,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T12:47:13,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T12:47:13,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T12:47:13,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T12:47:13,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T12:47:13,034 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T12:47:13,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-04T12:47:13,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-04T12:47:13,034 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2024-12-04T12:47:13,034 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T12:47:13,034 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T12:47:13,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-04T12:47:13,046 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {938b79ed23df:38879=1733316398288} newestTimestamps: {938b79ed23df:38879=1733316413641} 2024-12-04T12:47:13,049 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 938b79ed23df%2C38879%2C1733316387992.meta.1733316392177.meta 2024-12-04T12:47:13,049 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 938b79ed23df%2C38879%2C1733316387992.meta.1733316413617.meta 2024-12-04T12:47:13,049 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 938b79ed23df%2C38879%2C1733316387992.meta.1733316432943.meta 2024-12-04T12:47:13,049 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316413592 2024-12-04T12:47:13,049 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316432926 2024-12-04T12:47:13,049 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316413641 2024-12-04T12:47:13,049 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316432958 2024-12-04T12:47:13,061 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316413617.meta to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.meta.1733316413617.meta 2024-12-04T12:47:13,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741892_1068 (size=950) 2024-12-04T12:47:13,465 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1733316396597 target: hdfs://localhost:41555/backupUT/backup_1733316428515/default/test-1733316396597 descriptor: 'test-1733316396597', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:47:13,465 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-04T12:47:13,469 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1733316396597 2024-12-04T12:47:13,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741893_1069 (size=53) 2024-12-04T12:47:13,879 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1733316396597 2024-12-04T12:47:13,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741894_1070 (size=364) 2024-12-04T12:47:14,297 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17333163965971 target: hdfs://localhost:41555/backupUT/backup_1733316428515/ns2/test-17333163965971 descriptor: 'ns2:test-17333163965971', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:47:14,297 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-04T12:47:14,302 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17333163965971 2024-12-04T12:47:14,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741895_1071 (size=50) 2024-12-04T12:47:14,711 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17333163965971 2024-12-04T12:47:14,728 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:41555/backupUT/.tmp/backup_1733316428515 from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316413592;hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316413641 2024-12-04T12:47:14,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28cd9187, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:14,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:47:14,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:47:14,732 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:47:14,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:47:14,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:47:14,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c338d68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:14,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:47:14,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:47:14,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:14,733 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:47:14,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@231f5fc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:14,734 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:47:14,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:47:14,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@129e166b 2024-12-04T12:47:14,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:47:14,736 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:47:14,742 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-04T12:47:14,742 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17333163965971 2024-12-04T12:47:14,742 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17333163965971,, stopping at row=ns2:test-17333163965971 ,, for max=2147483647 with caching=100 2024-12-04T12:47:14,743 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:47:14,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:47:14,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:47:14,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:47:14,747 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17333163965971: ns2:test-17333163965971; 2024-12-04T12:47:14,748 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1733316396597 2024-12-04T12:47:14,748 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733316396597,, stopping at row=test-1733316396597 ,, for max=2147483647 with caching=100 2024-12-04T12:47:14,750 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1733316396597: test-1733316396597; 2024-12-04T12:47:14,750 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-12-04T12:47:14,750 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/partitions_4dc4150d-2aeb-4335-a3a6-88c1b77bcbdc 2024-12-04T12:47:14,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741896_1072 (size=192) 2024-12-04T12:47:15,182 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,183 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,183 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,964 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-11802061877902166713.jar 2024-12-04T12:47:15,964 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,964 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,965 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,965 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,965 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,965 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,965 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,965 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,966 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,966 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:47:15,966 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T12:47:15,966 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T12:47:15,966 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T12:47:15,967 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T12:47:15,967 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T12:47:15,967 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T12:47:15,967 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T12:47:15,967 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T12:47:15,968 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T12:47:15,968 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T12:47:15,969 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,969 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,969 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,969 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,970 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,970 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:15,970 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:15,970 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:test-17333163965971,default:test-1733316396597 2024-12-04T12:47:15,971 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:47:15,971 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:157) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:47:15,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:15,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:15,971 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:47:15,971 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:47:16,719 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-1036516161147467154.jar 2024-12-04T12:47:16,729 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:47:16,752 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316413592 for WAL files 2024-12-04T12:47:16,754 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316413592 2024-12-04T12:47:16,754 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316413641 for WAL files 2024-12-04T12:47:16,755 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316413641 2024-12-04T12:47:16,846 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local1154302058_0003_bef13e6a-6337-4d00-b561-7fa2e58b3725/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T12:47:16,846 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local1154302058_0003_bef13e6a-6337-4d00-b561-7fa2e58b3725/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T12:47:16,929 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316413641 for hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316413641 (-9223372036854775808:9223372036854775807) length:30490 2024-12-04T12:47:16,950 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-12-04T12:47:16,950 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-12-04T12:47:16,987 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316413592 for hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316413592 (-9223372036854775808:9223372036854775807) length:1198 2024-12-04T12:47:16,991 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-12-04T12:47:16,991 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-12-04T12:47:17,062 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f375fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:17,062 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:47:17,063 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:47:17,063 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:47:17,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:47:17,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:47:17,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e83b7e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:17,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:47:17,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:47:17,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:17,065 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:47:17,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3811ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:17,066 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:47:17,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:47:17,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:47:17,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:47:17,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17333163965971', row='row-t20', locateType=CURRENT is [region=ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:47:17,070 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-12-04T12:47:17,070 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:47:17,070 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:17,071 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:17,071 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:47:17,071 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-12-04T12:47:17,071 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 938b79ed23df 2024-12-04T12:47:17,075 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/backupUT/.tmp/backup_1733316428515/_temporary/0/_temporary/attempt_local1154302058_0003_r_000000_0/ns2/test-17333163965971/f/b3566a2a5ffe48c5a62f97543a623b22 is 38, key is row-t20/f:q1/1733316427166/Put/seqid=0 2024-12-04T12:47:17,078 WARN [Thread-950 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [938b79ed23df:38879] Specified favored nodes: [938b79ed23df:38879] 2024-12-04T12:47:17,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741897_1073 (size=5275) 2024-12-04T12:47:17,532 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c7a52a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:17,532 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:47:17,532 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:47:17,533 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:47:17,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:47:17,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:47:17,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@499dd74e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:17,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:47:17,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:47:17,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:17,535 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:47:17,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75c19038, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:17,536 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:47:17,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:47:17,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:47:17,539 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:47:17,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733316396597', row='row-test-1733316396597-1-0', locateType=CURRENT is [region=test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443., hostname=938b79ed23df,38879,1733316387992, seqNum=107] 2024-12-04T12:47:17,541 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-12-04T12:47:17,542 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:47:17,542 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:17,542 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:17,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:47:17,542 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733316396597-1-0] 2024-12-04T12:47:17,542 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 938b79ed23df 2024-12-04T12:47:17,552 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@242a66b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:17,552 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:47:17,552 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:47:17,553 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:47:17,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:47:17,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:47:17,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d62a37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:17,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:47:17,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:47:17,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:17,555 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:47:17,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@508d0369, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:17,556 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:47:17,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:47:17,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:47:17,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:47:17,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733316396597', row='row-test-1733316396597-3-0', locateType=CURRENT is [region=test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443., hostname=938b79ed23df,38879,1733316387992, seqNum=107] 2024-12-04T12:47:17,561 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-12-04T12:47:17,561 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:47:17,561 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:17,561 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:17,561 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:47:17,561 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733316396597-3-0] 2024-12-04T12:47:17,561 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 938b79ed23df 2024-12-04T12:47:17,570 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/backupUT/.tmp/backup_1733316428515/_temporary/0/_temporary/attempt_local1154302058_0003_r_000001_0/default/test-1733316396597/f/4a15e43ad10f4a0bbb2f3d7b6fce41a8 is 59, key is row-test-1733316396597-1-10/f:q1/1733316426782/Put/seqid=0 2024-12-04T12:47:17,572 WARN [Thread-962 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [938b79ed23df:38879] Specified favored nodes: [938b79ed23df:38879] 2024-12-04T12:47:17,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741898_1074 (size=10770) 2024-12-04T12:47:17,977 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/backupUT/.tmp/backup_1733316428515/_temporary/0/_temporary/attempt_local1154302058_0003_r_000001_0/default/test-1733316396597/mob/401b065e18594b77a35fd536309ea34e is 63, key is row-test-1733316396597-3-100/mob:q1/1733316427129/Put/seqid=0 2024-12-04T12:47:17,982 WARN [Thread-964 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [938b79ed23df:38879] Specified favored nodes: [938b79ed23df:38879] 2024-12-04T12:47:17,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741899_1075 (size=11811) 2024-12-04T12:47:18,895 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:41555/backupUT 2024-12-04T12:47:18,895 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1733316428515 2024-12-04T12:47:18,895 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-12-04T12:47:18,909 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:41555/backupUT/.tmp/backup_1733316428515, hdfs://localhost:41555/backupUT] 2024-12-04T12:47:18,980 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:47:19,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741900_1076 (size=5275) 2024-12-04T12:47:19,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741901_1077 (size=10770) 2024-12-04T12:47:19,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741902_1078 (size=11811) 2024-12-04T12:47:20,573 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-12-04T12:47:20,577 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1733316428515 2024-12-04T12:47:20,577 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 27856 bytes copied." 2024-12-04T12:47:20,577 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local248551221_0004 completed: true true 2024-12-04T12:47:20,579 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=3831857 FILE: Number of bytes written=6214117 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=231564 HDFS: Number of bytes written=2433152 HDFS: Number of read operations=646 HDFS: Number of large read operations=0 HDFS: Number of write operations=249 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=12 Map output records=0 Input split bytes=340 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=3004 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=27856 Bytes Copied=27856 Bytes Expected=27856 Files Copied=4 DIR_COPY=8 2024-12-04T12:47:20,579 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:41555/backupUT/.tmp/backup_1733316428515 to hdfs://localhost:41555/backupUT finished. 2024-12-04T12:47:20,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system_bulk', row='bulk:test-1733316396597:', locateType=CURRENT is [region=backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:47:20,599 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733316428515 2024-12-04T12:47:20,603 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733316406744} 2024-12-04T12:47:20,603 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-12-04T12:47:20,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741903_1079 (size=317) 2024-12-04T12:47:21,009 INFO [Time-limited test {}] impl.BackupManifest(504): Manifest file stored to hdfs://localhost:41555/backupUT/backup_1733316428515/.backup.manifest 2024-12-04T12:47:21,009 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1733316428515 finished: type=INCREMENTAL,tablelist=test-1733316396597;ns2:test-17333163965971,targetRootDir=hdfs://localhost:41555/backupUT,startts=1733316432795,completets=1733316440598,bytescopied=0 2024-12-04T12:47:21,009 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-04T12:47:21,016 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-12-04T12:47:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-04T12:47:21,021 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-04T12:47:21,024 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-04T12:47:21,028 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1733316428515 completed. 2024-12-04T12:47:21,028 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-12-04T12:47:21,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:21,049 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:41555/backupUT/backup_1733316428515/.backup.manifest 2024-12-04T12:47:21,052 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733316428515/.backup.manifest 2024-12-04T12:47:21,053 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:41555/backupUT/backup_1733316428515/.backup.manifest 2024-12-04T12:47:21,055 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733316428515/.backup.manifest 2024-12-04T12:47:21,056 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table test-1733316396597 from 'test-1733316396597', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1733316396597', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:47:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1733316396597 2024-12-04T12:47:21,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741904_1080 (size=908) 2024-12-04T12:47:21,471 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/.tabledesc/.tableinfo.0000000003.950 2024-12-04T12:47:21,471 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/.tabledesc/.tableinfo.0000000004.908 2024-12-04T12:47:21,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1733316396597}] 2024-12-04T12:47:21,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, REOPEN/MOVE}] 2024-12-04T12:47:21,477 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, REOPEN/MOVE 2024-12-04T12:47:21,478 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=87e3e36a673c628af1440164c1fd3443, regionState=CLOSING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:47:21,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, REOPEN/MOVE because future has completed 2024-12-04T12:47:21,481 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T12:47:21,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 87e3e36a673c628af1440164c1fd3443, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:47:21,633 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(122): Close 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:21,633 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T12:47:21,633 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1722): Closing 87e3e36a673c628af1440164c1fd3443, disabling compactions & flushes 2024-12-04T12:47:21,633 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1755): Closing region test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:47:21,633 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:47:21,633 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1843): Acquired close lock on test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. after waiting 0 ms 2024-12-04T12:47:21,633 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1853): Updates disabled for region test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:47:21,641 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=106 2024-12-04T12:47:21,642 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:47:21,642 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1973): Closed test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:47:21,642 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1676): Region close journal for 87e3e36a673c628af1440164c1fd3443: Waiting for close lock at 1733316441633Running coprocessor pre-close hooks at 1733316441633Disabling compacts and flushes for region at 1733316441633Disabling writes for close at 1733316441633Writing region close event to WAL at 1733316441635 (+2 ms)Running coprocessor post-close hooks at 1733316441642 (+7 ms)Closed at 1733316441642 2024-12-04T12:47:21,642 WARN [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegionServer(3299): Not adding moved region record: 87e3e36a673c628af1440164c1fd3443 to self. 2024-12-04T12:47:21,645 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(157): Closed 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:21,645 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=87e3e36a673c628af1440164c1fd3443, regionState=CLOSED 2024-12-04T12:47:21,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 87e3e36a673c628af1440164c1fd3443, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:47:21,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-04T12:47:21,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseRegionProcedure 87e3e36a673c628af1440164c1fd3443, server=938b79ed23df,38879,1733316387992 in 167 msec 2024-12-04T12:47:21,651 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, REOPEN/MOVE; state=CLOSED, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=true 2024-12-04T12:47:21,802 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=87e3e36a673c628af1440164c1fd3443, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:47:21,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, REOPEN/MOVE because future has completed 2024-12-04T12:47:21,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87e3e36a673c628af1440164c1fd3443, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:47:21,972 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:47:21,972 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 87e3e36a673c628af1440164c1fd3443, NAME => 'test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443.', STARTKEY => '', ENDKEY => ''} 2024-12-04T12:47:21,973 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:47:21,973 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1733316396597 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:21,973 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:47:21,974 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:21,974 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:21,976 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:21,977 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87e3e36a673c628af1440164c1fd3443 columnFamilyName f 2024-12-04T12:47:21,978 DEBUG [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:47:21,987 DEBUG [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40 2024-12-04T12:47:21,992 DEBUG [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/ed2773986acc4ae18188f34dec668da1 2024-12-04T12:47:21,993 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.HStore(327): Store=87e3e36a673c628af1440164c1fd3443/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:47:21,993 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:21,994 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87e3e36a673c628af1440164c1fd3443 columnFamilyName f2 2024-12-04T12:47:21,994 DEBUG [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:47:21,995 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.HStore(327): Store=87e3e36a673c628af1440164c1fd3443/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:47:21,995 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:21,996 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87e3e36a673c628af1440164c1fd3443 columnFamilyName mob 2024-12-04T12:47:21,997 DEBUG [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:47:22,003 DEBUG [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/5799223a4841435fb10d4958b6b73deb 2024-12-04T12:47:22,007 DEBUG [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/6759befaa3d44e67a2be163409a2ab50 2024-12-04T12:47:22,007 INFO [StoreOpener-87e3e36a673c628af1440164c1fd3443-1 {}] regionserver.HStore(327): Store=87e3e36a673c628af1440164c1fd3443/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:47:22,008 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:22,008 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:22,010 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:22,011 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:22,011 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:22,011 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1733316396597 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T12:47:22,012 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:22,013 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 87e3e36a673c628af1440164c1fd3443; next sequenceid=338; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10447029600, jitterRate=-0.027044549584388733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T12:47:22,013 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:22,015 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 87e3e36a673c628af1440164c1fd3443: Running coprocessor pre-open hook at 1733316441974Writing region info on filesystem at 1733316441974Initializing all the Stores at 1733316441975 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316441975Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316441976 (+1 ms)Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316441976Cleaning up temporary data from old regions at 1733316442011 (+35 ms)Running coprocessor post-open hooks at 1733316442013 (+2 ms)Region opened successfully at 1733316442015 (+2 ms) 2024-12-04T12:47:22,016 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443., pid=46, masterSystemTime=1733316441962 2024-12-04T12:47:22,018 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:47:22,018 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:47:22,019 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=87e3e36a673c628af1440164c1fd3443, regionState=OPEN, openSeqNum=338, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:47:22,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87e3e36a673c628af1440164c1fd3443, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:47:22,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=44 2024-12-04T12:47:22,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=44, state=SUCCESS, hasLock=false; OpenRegionProcedure 87e3e36a673c628af1440164c1fd3443, server=938b79ed23df,38879,1733316387992 in 213 msec 2024-12-04T12:47:22,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-12-04T12:47:22,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1733316396597, region=87e3e36a673c628af1440164c1fd3443, REOPEN/MOVE in 547 msec 2024-12-04T12:47:22,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-04T12:47:22,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1733316396597 in 552 msec 2024-12-04T12:47:22,030 DEBUG [PEWorker-5 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=test-1733316396597 2024-12-04T12:47:22,036 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f3/027afb5687a04f3cb770de8c58f62ade to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f3/027afb5687a04f3cb770de8c58f62ade 2024-12-04T12:47:22,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1733316396597 in 980 msec 2024-12-04T12:47:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T12:47:22,040 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1733316396597 completed 2024-12-04T12:47:22,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:22,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:22,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:22,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:22,079 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-04T12:47:22,083 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733316442040 2024-12-04T12:47:22,085 DEBUG [Time-limited test {}] impl.TableBackupClient(330): Dependent incremental backup image: {BackupID=backup_1733316428515} 2024-12-04T12:47:22,085 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733316406744} 2024-12-04T12:47:22,085 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 2 ancestors for the current backup. 2024-12-04T12:47:22,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:22,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:22,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:22,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:22,131 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected Exception : Mismatch in column family descriptor for table: test-1733316396597 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException: Mismatch in column family descriptor for table: test-1733316396597 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException$ColumnFamilyMismatchExceptionBuilder.build(ColumnFamilyMismatchException.java:56) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.verifyCfCompatibility(IncrementalTableBackupClient.java:513) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:280) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.lambda$TestIncBackupRestore$0(TestIncrementalBackup.java:175) ~[test-classes/:?] at org.junit.Assert.assertThrows(Assert.java:1001) ~[junit-4.13.2.jar:4.13.2] at org.junit.Assert.assertThrows(Assert.java:981) ~[junit-4.13.2.jar:4.13.2] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:174) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T12:47:22,136 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1733316442040,startts=0,failedts=1733316442132,failedphase=REQUEST,failedmessage= Mismatch in column family descriptor for table: test-1733316396597 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'tru 2024-12-04T12:47:22,136 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-12-04T12:47:22,138 WARN [Time-limited test {}] impl.BackupSystemTable(1549): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2024-12-04T12:47:22,138 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-04T12:47:22,140 ERROR [Time-limited test {}] impl.BackupSystemTable(1578): Snapshot snapshot_backup_system does not exists 2024-12-04T12:47:22,140 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: REQUEST 2024-12-04T12:47:22,142 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-04T12:47:22,144 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1733316442040 failed. 2024-12-04T12:47:22,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:22,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:22,163 INFO [Time-limited test {}] impl.BackupManager(225): Full backup all the tables available in the cluster: [backup:system_bulk, test-1733316396597, ns2:test-17333163965971, ns3:test-17333163965972, ns4:test-17333163965973] 2024-12-04T12:47:22,163 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-04T12:47:22,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T12:47:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733316442168 (current time:1733316442168). 2024-12-04T12:47:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T12:47:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-04T12:47:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109b6b09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:47:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:47:22,171 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:47:22,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:47:22,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:47:22,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e5c20ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:22,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:47:22,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:47:22,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:22,172 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:47:22,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ee93d24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:47:22,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:47:22,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:47:22,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:47:22,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049. 2024-12-04T12:47:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:47:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:22,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:47:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T12:47:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T12:47:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-12-04T12:47:22,180 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T12:47:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T12:47:22,181 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T12:47:22,183 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T12:47:22,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741905_1081 (size=52) 2024-12-04T12:47:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T12:47:22,430 DEBUG [master/938b79ed23df:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733316396597 2024-12-04T12:47:22,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741906_1082 (size=950) 2024-12-04T12:47:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T12:47:22,591 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T12:47:22,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949}] 2024-12-04T12:47:22,592 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=49, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:47:22,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38879 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=49 2024-12-04T12:47:22,746 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:47:22,747 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 9dab1d763bdadc4adf3b254f1a913949 2/2 column families, dataSize=2.61 KB heapSize=3.99 KB 2024-12-04T12:47:22,756 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/meta/bb0f881d4a4a457486fe958ca934bea8 is 158, key is trslm:hdfs://localhost:41555/backupUT\x00ns2:test-17333163965971/meta:log-roll-map/1733316440585/Put/seqid=0 2024-12-04T12:47:22,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741907_1083 (size=5745) 2024-12-04T12:47:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T12:47:22,851 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/.tabledesc/.tableinfo.0000000004.908 2024-12-04T12:47:22,851 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/.tabledesc/.tableinfo.0000000005.950 2024-12-04T12:47:22,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733316396597 in 423 msec 2024-12-04T12:47:23,162 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=492 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/meta/bb0f881d4a4a457486fe958ca934bea8 2024-12-04T12:47:23,177 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/session/156393e147e744b0aa829f60f1c74c5e is 1245, key is session:backup_1733316442040/session:context/1733316442141/Put/seqid=0 2024-12-04T12:47:23,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741908_1084 (size=6682) 2024-12-04T12:47:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T12:47:23,582 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/session/156393e147e744b0aa829f60f1c74c5e 2024-12-04T12:47:23,594 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/meta/bb0f881d4a4a457486fe958ca934bea8 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/bb0f881d4a4a457486fe958ca934bea8 2024-12-04T12:47:23,601 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/bb0f881d4a4a457486fe958ca934bea8, entries=4, sequenceid=31, filesize=5.6 K 2024-12-04T12:47:23,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/session/156393e147e744b0aa829f60f1c74c5e as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/156393e147e744b0aa829f60f1c74c5e 2024-12-04T12:47:23,608 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/156393e147e744b0aa829f60f1c74c5e, entries=3, sequenceid=31, filesize=6.5 K 2024-12-04T12:47:23,609 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~2.61 KB/2669, heapSize ~3.96 KB/4056, currentSize=0 B/0 for 9dab1d763bdadc4adf3b254f1a913949 in 862ms, sequenceid=31, compaction requested=true 2024-12-04T12:47:23,609 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 9dab1d763bdadc4adf3b254f1a913949: 2024-12-04T12:47:23,609 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. for snapshot_backup_system completed. 2024-12-04T12:47:23,609 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949.' region-info for snapshot=snapshot_backup_system 2024-12-04T12:47:23,609 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T12:47:23,609 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/6bb6c66690d84dfb8bc5bd7b77d4a104, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/bb0f881d4a4a457486fe958ca934bea8] hfiles 2024-12-04T12:47:23,610 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/6bb6c66690d84dfb8bc5bd7b77d4a104 for snapshot=snapshot_backup_system 2024-12-04T12:47:23,610 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/bb0f881d4a4a457486fe958ca934bea8 for snapshot=snapshot_backup_system 2024-12-04T12:47:23,610 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/0a06d46ae887482d95f1e19410656caa, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/5d1327784fd34e43a68805beada718ac, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/156393e147e744b0aa829f60f1c74c5e] hfiles 2024-12-04T12:47:23,610 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/0a06d46ae887482d95f1e19410656caa for snapshot=snapshot_backup_system 2024-12-04T12:47:23,610 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/5d1327784fd34e43a68805beada718ac for snapshot=snapshot_backup_system 2024-12-04T12:47:23,610 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/156393e147e744b0aa829f60f1c74c5e for snapshot=snapshot_backup_system 2024-12-04T12:47:23,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741909_1085 (size=251) 2024-12-04T12:47:24,016 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:47:24,016 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-04T12:47:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-04T12:47:24,017 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:47:24,017 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=49, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:47:24,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=47 2024-12-04T12:47:24,021 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T12:47:24,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=47, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949 in 1.4270 sec 2024-12-04T12:47:24,023 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T12:47:24,024 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T12:47:24,024 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-04T12:47:24,026 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-04T12:47:24,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741910_1086 (size=888) 2024-12-04T12:47:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T12:47:24,442 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T12:47:24,452 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T12:47:24,453 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_backup_system 2024-12-04T12:47:24,454 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T12:47:24,455 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-12-04T12:47:24,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2770 sec 2024-12-04T12:47:25,582 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T12:47:25,588 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region c988c84788a01c971b5789026625491a, had cached 0 bytes from a total of 8413 2024-12-04T12:47:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T12:47:26,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-04T12:47:26,336 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733316442145 started at 1733316446335. 2024-12-04T12:47:26,341 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733316442145 has been started. 2024-12-04T12:47:26,346 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-12-04T12:47:26,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-12-04T12:47:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-04T12:47:26,347 INFO [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-04T12:47:26,347 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-04T12:47:26,347 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-04T12:47:26,347 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-04T12:47:26,348 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T12:47:26,348 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T12:47:26,399 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T12:47:26,400 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T12:47:26,400 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T12:47:26,401 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,401 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-04T12:47:26,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-04T12:47:26,403 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T12:47:26,405 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-04T12:47:26,405 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T12:47:26,405 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-04T12:47:26,406 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-04T12:47:26,406 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-04T12:47:26,406 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-04T12:47:26,406 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-04T12:47:26,406 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-04T12:47:26,406 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-04T12:47:26,406 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '938b79ed23df,38879,1733316387992' joining acquired barrier for procedure (rolllog) in zk 2024-12-04T12:47:26,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,415 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-04T12:47:26,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T12:47:26,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T12:47:26,416 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-04T12:47:26,416 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-04T12:47:26,417 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T12:47:26,417 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T12:47:26,418 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:47:26,419 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,419 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T12:47:26,420 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '938b79ed23df,38879,1733316387992' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-04T12:47:26,420 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@5039cf4f[Count = 0] remaining members to acquire global barrier 2024-12-04T12:47:26,420 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-04T12:47:26,420 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-04T12:47:26,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T12:47:26,424 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-04T12:47:26,424 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-04T12:47:26,424 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-04T12:47:26,424 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,424 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-04T12:47:26,424 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-04T12:47:26,424 DEBUG [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,425 INFO [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733316432926 highest: 1733316432958 on 938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,425 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0:(num 1733316432926) roll requested 2024-12-04T12:47:26,440 DEBUG [regionserver/938b79ed23df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316446425, exclude list is [], retry=0 2024-12-04T12:47:26,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:47:26,445 INFO [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316432926 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316446425 2024-12-04T12:47:26,445 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:47:26,445 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316432926 is not closed yet, will try archiving it next time 2024-12-04T12:47:26,445 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 938b79ed23df%2C38879%2C1733316387992.meta:.meta(num 1733316432943) roll requested 2024-12-04T12:47:26,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741889_1065 (size=93) 2024-12-04T12:47:26,446 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316432926 not finished, retry = 0 2024-12-04T12:47:26,457 DEBUG [regionserver/938b79ed23df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316446445.meta, exclude list is [], retry=0 2024-12-04T12:47:26,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:47:26,461 INFO [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316432943.meta with entries=4, filesize=1.85 KB; new WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316446445.meta 2024-12-04T12:47:26,461 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:47:26,461 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316432943.meta is not closed yet, will try archiving it next time 2024-12-04T12:47:26,461 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1:(num 1733316432958) roll requested 2024-12-04T12:47:26,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741890_1066 (size=1903) 2024-12-04T12:47:26,462 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316432943.meta not finished, retry = 0 2024-12-04T12:47:26,474 DEBUG [regionserver/938b79ed23df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316446461, exclude list is [], retry=0 2024-12-04T12:47:26,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:47:26,477 INFO [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316432958 with entries=15, filesize=4.80 KB; new WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316446461 2024-12-04T12:47:26,478 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:47:26,478 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316432958 is not closed yet, will try archiving it next time 2024-12-04T12:47:26,478 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316413641 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316413641 2024-12-04T12:47:26,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741891_1067 (size=4927) 2024-12-04T12:47:26,479 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316432958 not finished, retry = 0 2024-12-04T12:47:26,485 DEBUG [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-12-04T12:47:26,485 INFO [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733316446425 on 938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:26,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:26,507 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-04T12:47:26,507 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '938b79ed23df,38879,1733316387992' in zk 2024-12-04T12:47:26,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,515 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-04T12:47:26,515 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T12:47:26,515 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-04T12:47:26,515 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,515 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T12:47:26,515 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T12:47:26,516 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T12:47:26,517 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T12:47:26,517 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:47:26,517 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,518 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T12:47:26,518 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:47:26,519 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,520 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '938b79ed23df,38879,1733316387992': 2024-12-04T12:47:26,520 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '938b79ed23df,38879,1733316387992' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-04T12:47:26,520 INFO [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-04T12:47:26,520 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-04T12:47:26,520 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-04T12:47:26,520 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-04T12:47:26,520 INFO [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-04T12:47:26,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T12:47:26,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T12:47:26,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T12:47:26,531 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-04T12:47:26,531 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-04T12:47:26,532 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T12:47:26,532 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T12:47:26,532 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T12:47:26,532 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T12:47:26,532 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T12:47:26,532 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T12:47:26,533 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,533 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:47:26,533 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T12:47:26,533 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T12:47:26,534 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:47:26,534 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,535 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,535 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T12:47:26,535 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:47:26,536 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T12:47:26,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T12:47:26,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T12:47:26,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:47:26,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T12:47:26,541 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T12:47:26,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T12:47:26,541 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T12:47:26,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T12:47:26,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T12:47:26,541 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T12:47:26,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-04T12:47:26,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-04T12:47:26,541 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T12:47:26,542 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T12:47:26,547 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316432926 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316432926 2024-12-04T12:47:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-04T12:47:26,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733316446556_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-12-04T12:47:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733316446557 (current time:1733316446557). 2024-12-04T12:47:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T12:47:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733316446556_backup_system_bulk VERSION not specified, setting to 2 2024-12-04T12:47:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7088cab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:47:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:47:26,558 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:47:26,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:47:26,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:47:26,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4836439a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:26,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:47:26,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:47:26,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:26,560 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:47:26,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f46fc2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:47:26,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:47:26,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:47:26,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:47:26,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049. 2024-12-04T12:47:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:47:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:26,564 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:47:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T12:47:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733316446556_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-12-04T12:47:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733316446556_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-12-04T12:47:26,566 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733316446556_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T12:47:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T12:47:26,568 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733316446556_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T12:47:26,570 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733316446556_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T12:47:26,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741914_1090 (size=76) 2024-12-04T12:47:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T12:47:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T12:47:26,980 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733316446556_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T12:47:26,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 58f768c3c54039235f5a6360aacc8ebf}] 2024-12-04T12:47:26,983 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:47:27,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38879 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2024-12-04T12:47:27,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. 2024-12-04T12:47:27,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 58f768c3c54039235f5a6360aacc8ebf: 2024-12-04T12:47:27,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. for snapshot_1733316446556_backup_system_bulk completed. 2024-12-04T12:47:27,138 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(241): Storing 'backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf.' region-info for snapshot=snapshot_1733316446556_backup_system_bulk 2024-12-04T12:47:27,138 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T12:47:27,138 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-04T12:47:27,138 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-04T12:47:27,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741915_1091 (size=61) 2024-12-04T12:47:27,169 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-12-04T12:47:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T12:47:27,549 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. 2024-12-04T12:47:27,550 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-04T12:47:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-04T12:47:27,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733316446556_backup_system_bulk on region 58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:47:27,551 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:47:27,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-04T12:47:27,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 58f768c3c54039235f5a6360aacc8ebf in 584 msec 2024-12-04T12:47:27,571 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733316446556_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T12:47:27,573 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733316446556_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T12:47:27,574 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733316446556_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T12:47:27,575 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733316446556_backup_system_bulk 2024-12-04T12:47:27,576 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_1733316446556_backup_system_bulk 2024-12-04T12:47:27,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741916_1092 (size=702) 2024-12-04T12:47:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T12:47:27,991 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733316446556_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T12:47:28,000 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733316446556_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T12:47:28,001 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_1733316446556_backup_system_bulk to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_1733316446556_backup_system_bulk 2024-12-04T12:47:28,002 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733316446556_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T12:47:28,002 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733316446556_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-12-04T12:47:28,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733316446556_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } in 1.4370 sec 2024-12-04T12:47:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T12:47:28,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system_bulk completed 2024-12-04T12:47:28,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733316448726_ns3_test-17333163965972 table=ns3:test-17333163965972 type=FLUSH ttl=0 } 2024-12-04T12:47:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733316448728 (current time:1733316448728). 2024-12-04T12:47:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T12:47:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733316448726_ns3_test-17333163965972 VERSION not specified, setting to 2 2024-12-04T12:47:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@318b91f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:47:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:47:28,733 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:47:28,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:47:28,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:47:28,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38db2463, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:28,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:47:28,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:47:28,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:28,735 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:47:28,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@172bcd5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:47:28,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:47:28,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:47:28,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:47:28,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049. 2024-12-04T12:47:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:47:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:28,740 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:47:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T12:47:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733316448726_ns3_test-17333163965972 table=ns3:test-17333163965972 type=FLUSH ttl=0 } 2024-12-04T12:47:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733316448726_ns3_test-17333163965972 table=ns3:test-17333163965972 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-12-04T12:47:28,743 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733316448726_ns3_test-17333163965972 table=ns3:test-17333163965972 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T12:47:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T12:47:28,745 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733316448726_ns3_test-17333163965972 table=ns3:test-17333163965972 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T12:47:28,747 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733316448726_ns3_test-17333163965972 table=ns3:test-17333163965972 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T12:47:28,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741917_1093 (size=86) 2024-12-04T12:47:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T12:47:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T12:47:29,154 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733316448726_ns3_test-17333163965972 table=ns3:test-17333163965972 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T12:47:29,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d7d9cf8458b0e699dff32ea7d56f7d9d}] 2024-12-04T12:47:29,156 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d7d9cf8458b0e699dff32ea7d56f7d9d 2024-12-04T12:47:29,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38879 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2024-12-04T12:47:29,310 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. 2024-12-04T12:47:29,310 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for d7d9cf8458b0e699dff32ea7d56f7d9d: 2024-12-04T12:47:29,310 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. for snapshot_1733316448726_ns3_test-17333163965972 completed. 2024-12-04T12:47:29,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(241): Storing 'ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d.' region-info for snapshot=snapshot_1733316448726_ns3_test-17333163965972 2024-12-04T12:47:29,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T12:47:29,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-04T12:47:29,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741918_1094 (size=52) 2024-12-04T12:47:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T12:47:29,724 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. 2024-12-04T12:47:29,725 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-04T12:47:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-04T12:47:29,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733316448726_ns3_test-17333163965972 on region d7d9cf8458b0e699dff32ea7d56f7d9d 2024-12-04T12:47:29,727 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d7d9cf8458b0e699dff32ea7d56f7d9d 2024-12-04T12:47:29,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-04T12:47:29,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; SnapshotRegionProcedure d7d9cf8458b0e699dff32ea7d56f7d9d in 576 msec 2024-12-04T12:47:29,734 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733316448726_ns3_test-17333163965972 table=ns3:test-17333163965972 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T12:47:29,735 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733316448726_ns3_test-17333163965972 table=ns3:test-17333163965972 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T12:47:29,736 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733316448726_ns3_test-17333163965972 table=ns3:test-17333163965972 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T12:47:29,736 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733316448726_ns3_test-17333163965972 2024-12-04T12:47:29,737 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_1733316448726_ns3_test-17333163965972 2024-12-04T12:47:29,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741919_1095 (size=443) 2024-12-04T12:47:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T12:47:30,155 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733316448726_ns3_test-17333163965972 table=ns3:test-17333163965972 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T12:47:30,164 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733316448726_ns3_test-17333163965972 table=ns3:test-17333163965972 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T12:47:30,165 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_1733316448726_ns3_test-17333163965972 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_1733316448726_ns3_test-17333163965972 2024-12-04T12:47:30,167 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733316448726_ns3_test-17333163965972 table=ns3:test-17333163965972 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T12:47:30,167 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733316448726_ns3_test-17333163965972 table=ns3:test-17333163965972 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-12-04T12:47:30,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733316448726_ns3_test-17333163965972 table=ns3:test-17333163965972 type=FLUSH ttl=0 } in 1.4260 sec 2024-12-04T12:47:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T12:47:30,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns3:test-17333163965972 completed 2024-12-04T12:47:30,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733316450904_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } 2024-12-04T12:47:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733316450904 (current time:1733316450904). 2024-12-04T12:47:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T12:47:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733316450904_default_test-1733316396597 VERSION not specified, setting to 2 2024-12-04T12:47:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e6ec7cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:47:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:47:30,906 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:47:30,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:47:30,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:47:30,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74951f44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:30,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:47:30,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:47:30,908 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:30,908 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:47:30,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@676e489, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:47:30,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:47:30,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:47:30,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:47:30,912 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049. 2024-12-04T12:47:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:47:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:30,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:47:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T12:47:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733316450904_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } 2024-12-04T12:47:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733316450904_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-12-04T12:47:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T12:47:30,915 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733316450904_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T12:47:30,916 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733316450904_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T12:47:30,918 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733316450904_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T12:47:30,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741920_1096 (size=84) 2024-12-04T12:47:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T12:47:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T12:47:31,328 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733316450904_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T12:47:31,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 87e3e36a673c628af1440164c1fd3443}] 2024-12-04T12:47:31,332 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:31,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38879 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2024-12-04T12:47:31,487 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:47:31,488 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 87e3e36a673c628af1440164c1fd3443: 2024-12-04T12:47:31,488 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. for snapshot_1733316450904_default_test-1733316396597 completed. 2024-12-04T12:47:31,488 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(241): Storing 'test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443.' region-info for snapshot=snapshot_1733316450904_default_test-1733316396597 2024-12-04T12:47:31,488 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T12:47:31,489 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/ed2773986acc4ae18188f34dec668da1] hfiles 2024-12-04T12:47:31,489 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40 for snapshot=snapshot_1733316450904_default_test-1733316396597 2024-12-04T12:47:31,489 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/ed2773986acc4ae18188f34dec668da1 for snapshot=snapshot_1733316450904_default_test-1733316396597 2024-12-04T12:47:31,489 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-04T12:47:31,489 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/6759befaa3d44e67a2be163409a2ab50, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/5799223a4841435fb10d4958b6b73deb] hfiles 2024-12-04T12:47:31,489 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/6759befaa3d44e67a2be163409a2ab50 for snapshot=snapshot_1733316450904_default_test-1733316396597 2024-12-04T12:47:31,489 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/5799223a4841435fb10d4958b6b73deb for snapshot=snapshot_1733316450904_default_test-1733316396597 2024-12-04T12:47:31,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741921_1097 (size=224) 2024-12-04T12:47:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T12:47:31,902 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:47:31,902 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-04T12:47:31,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-04T12:47:31,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733316450904_default_test-1733316396597 on region 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:31,905 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:31,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-04T12:47:31,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 87e3e36a673c628af1440164c1fd3443 in 578 msec 2024-12-04T12:47:31,913 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733316450904_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T12:47:31,914 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733316450904_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T12:47:31,915 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-12-04T12:47:31,916 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-12-04T12:47:31,916 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:47:31,916 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-12-04T12:47:31,916 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:47:31,917 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f2 2024-12-04T12:47:31,917 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:47:31,918 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/mobdir/data/default/test-1733316396597/64682aa8ca2d45d6ee9ace61337cabc4/mob/d41d8cd98f00b204e9800998ecf8427e20241204f4e86055bf4b405ab07a7b9c2fae1c0b_87e3e36a673c628af1440164c1fd3443] hfiles 2024-12-04T12:47:31,919 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/1): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/mobdir/data/default/test-1733316396597/64682aa8ca2d45d6ee9ace61337cabc4/mob/d41d8cd98f00b204e9800998ecf8427e20241204f4e86055bf4b405ab07a7b9c2fae1c0b_87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:31,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741922_1098 (size=168) 2024-12-04T12:47:32,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T12:47:32,329 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733316450904_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T12:47:32,329 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733316450904_default_test-1733316396597 2024-12-04T12:47:32,330 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_1733316450904_default_test-1733316396597 2024-12-04T12:47:32,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741923_1099 (size=1347) 2024-12-04T12:47:32,751 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733316450904_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T12:47:32,762 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733316450904_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T12:47:32,762 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_1733316450904_default_test-1733316396597 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_1733316450904_default_test-1733316396597 2024-12-04T12:47:32,765 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733316450904_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T12:47:32,765 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733316450904_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-12-04T12:47:32,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733316450904_default_test-1733316396597 table=test-1733316396597 type=FLUSH ttl=0 } in 1.8520 sec 2024-12-04T12:47:33,071 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 9dab1d763bdadc4adf3b254f1a913949, had cached 0 bytes from a total of 28925 2024-12-04T12:47:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T12:47:33,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1733316396597 completed 2024-12-04T12:47:33,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733316453075_ns4_test-17333163965973 table=ns4:test-17333163965973 type=FLUSH ttl=0 } 2024-12-04T12:47:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733316453077 (current time:1733316453077). 2024-12-04T12:47:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T12:47:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733316453075_ns4_test-17333163965973 VERSION not specified, setting to 2 2024-12-04T12:47:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@513bace2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:47:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:47:33,082 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:47:33,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:47:33,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:47:33,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f095f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:33,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:47:33,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:47:33,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:33,083 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:47:33,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16495a26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:47:33,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:47:33,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:47:33,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:47:33,088 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049. 2024-12-04T12:47:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:47:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:33,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:47:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T12:47:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733316453075_ns4_test-17333163965973 table=ns4:test-17333163965973 type=FLUSH ttl=0 } 2024-12-04T12:47:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733316453075_ns4_test-17333163965973 table=ns4:test-17333163965973 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-12-04T12:47:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T12:47:33,091 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733316453075_ns4_test-17333163965973 table=ns4:test-17333163965973 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T12:47:33,093 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733316453075_ns4_test-17333163965973 table=ns4:test-17333163965973 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T12:47:33,096 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733316453075_ns4_test-17333163965973 table=ns4:test-17333163965973 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T12:47:33,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741924_1100 (size=86) 2024-12-04T12:47:33,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T12:47:33,275 INFO [master/938b79ed23df:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T12:47:33,275 INFO [master/938b79ed23df:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T12:47:33,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T12:47:33,504 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733316453075_ns4_test-17333163965973 table=ns4:test-17333163965973 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T12:47:33,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure deae8c07fc7147a6340e830c87bd97da}] 2024-12-04T12:47:33,507 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:47:33,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38879 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2024-12-04T12:47:33,661 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. 2024-12-04T12:47:33,661 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for deae8c07fc7147a6340e830c87bd97da: 2024-12-04T12:47:33,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. for snapshot_1733316453075_ns4_test-17333163965973 completed. 2024-12-04T12:47:33,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(241): Storing 'ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da.' region-info for snapshot=snapshot_1733316453075_ns4_test-17333163965973 2024-12-04T12:47:33,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T12:47:33,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-04T12:47:33,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741925_1101 (size=52) 2024-12-04T12:47:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T12:47:34,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. 2024-12-04T12:47:34,074 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-04T12:47:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-04T12:47:34,075 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733316453075_ns4_test-17333163965973 on region deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:47:34,076 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:47:34,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-04T12:47:34,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; SnapshotRegionProcedure deae8c07fc7147a6340e830c87bd97da in 575 msec 2024-12-04T12:47:34,083 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733316453075_ns4_test-17333163965973 table=ns4:test-17333163965973 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T12:47:34,084 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733316453075_ns4_test-17333163965973 table=ns4:test-17333163965973 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T12:47:34,085 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733316453075_ns4_test-17333163965973 table=ns4:test-17333163965973 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T12:47:34,085 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733316453075_ns4_test-17333163965973 2024-12-04T12:47:34,085 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_1733316453075_ns4_test-17333163965973 2024-12-04T12:47:34,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741926_1102 (size=443) 2024-12-04T12:47:34,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T12:47:34,500 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733316453075_ns4_test-17333163965973 table=ns4:test-17333163965973 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T12:47:34,508 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733316453075_ns4_test-17333163965973 table=ns4:test-17333163965973 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T12:47:34,508 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_1733316453075_ns4_test-17333163965973 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_1733316453075_ns4_test-17333163965973 2024-12-04T12:47:34,510 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733316453075_ns4_test-17333163965973 table=ns4:test-17333163965973 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T12:47:34,510 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733316453075_ns4_test-17333163965973 table=ns4:test-17333163965973 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-12-04T12:47:34,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733316453075_ns4_test-17333163965973 table=ns4:test-17333163965973 type=FLUSH ttl=0 } in 1.4210 sec 2024-12-04T12:47:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T12:47:35,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns4:test-17333163965973 completed 2024-12-04T12:47:35,238 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733316455236_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } 2024-12-04T12:47:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733316455238 (current time:1733316455238). 2024-12-04T12:47:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T12:47:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733316455236_ns2_test-17333163965971 VERSION not specified, setting to 2 2024-12-04T12:47:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b3783fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:47:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:47:35,243 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:47:35,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:47:35,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:47:35,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@206eb219, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:35,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:47:35,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:47:35,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:35,245 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:47:35,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7224d28e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:35,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:47:35,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:47:35,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:47:35,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:47:35,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049. 2024-12-04T12:47:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:47:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:35,249 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:47:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T12:47:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733316455236_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } 2024-12-04T12:47:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733316455236_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-12-04T12:47:35,251 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733316455236_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T12:47:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T12:47:35,252 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733316455236_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T12:47:35,254 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733316455236_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T12:47:35,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741927_1103 (size=86) 2024-12-04T12:47:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T12:47:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T12:47:35,664 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733316455236_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T12:47:35,664 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c988c84788a01c971b5789026625491a}] 2024-12-04T12:47:35,666 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c988c84788a01c971b5789026625491a 2024-12-04T12:47:35,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38879 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2024-12-04T12:47:35,820 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:47:35,821 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing c988c84788a01c971b5789026625491a 1/1 column families, dataSize=170 B heapSize=816 B 2024-12-04T12:47:35,831 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/.tmp/f/9fe180fa04404396b0429daa9427c60f is 38, key is row-t20/f:q1/1733316427166/Put/seqid=0 2024-12-04T12:47:35,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741928_1104 (size=5134) 2024-12-04T12:47:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T12:47:36,238 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/.tmp/f/9fe180fa04404396b0429daa9427c60f 2024-12-04T12:47:36,251 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/.tmp/f/9fe180fa04404396b0429daa9427c60f as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/9fe180fa04404396b0429daa9427c60f 2024-12-04T12:47:36,259 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/9fe180fa04404396b0429daa9427c60f, entries=5, sequenceid=111, filesize=5.0 K 2024-12-04T12:47:36,260 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for c988c84788a01c971b5789026625491a in 439ms, sequenceid=111, compaction requested=false 2024-12-04T12:47:36,260 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for c988c84788a01c971b5789026625491a: 2024-12-04T12:47:36,260 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. for snapshot_1733316455236_ns2_test-17333163965971 completed. 2024-12-04T12:47:36,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a.' region-info for snapshot=snapshot_1733316455236_ns2_test-17333163965971 2024-12-04T12:47:36,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T12:47:36,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/194e38d3f9c54ed88dcc2848906ca7b4, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/9fe180fa04404396b0429daa9427c60f] hfiles 2024-12-04T12:47:36,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/194e38d3f9c54ed88dcc2848906ca7b4 for snapshot=snapshot_1733316455236_ns2_test-17333163965971 2024-12-04T12:47:36,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/9fe180fa04404396b0429daa9427c60f for snapshot=snapshot_1733316455236_ns2_test-17333163965971 2024-12-04T12:47:36,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741929_1105 (size=130) 2024-12-04T12:47:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T12:47:36,669 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:47:36,669 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-04T12:47:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-04T12:47:36,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733316455236_ns2_test-17333163965971 on region c988c84788a01c971b5789026625491a 2024-12-04T12:47:36,672 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c988c84788a01c971b5789026625491a 2024-12-04T12:47:36,678 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-04T12:47:36,679 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733316455236_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T12:47:36,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SnapshotRegionProcedure c988c84788a01c971b5789026625491a in 1.0100 sec 2024-12-04T12:47:36,681 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733316455236_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T12:47:36,682 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733316455236_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T12:47:36,682 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733316455236_ns2_test-17333163965971 2024-12-04T12:47:36,683 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_1733316455236_ns2_test-17333163965971 2024-12-04T12:47:36,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741930_1106 (size=496) 2024-12-04T12:47:37,096 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733316455236_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T12:47:37,107 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733316455236_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T12:47:37,107 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_1733316455236_ns2_test-17333163965971 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_1733316455236_ns2_test-17333163965971 2024-12-04T12:47:37,109 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733316455236_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T12:47:37,109 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733316455236_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-12-04T12:47:37,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733316455236_ns2_test-17333163965971 table=ns2:test-17333163965971 type=FLUSH ttl=0 } in 1.8590 sec 2024-12-04T12:47:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T12:47:37,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17333163965971 completed 2024-12-04T12:47:37,406 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1733316442145 2024-12-04T12:47:37,406 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-12-04T12:47:37,406 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 5 snapshots to be copied. 2024-12-04T12:47:37,406 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733316442145_backup:system_bulk 2024-12-04T12:47:37,406 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733316446556_backup_system_bulk to hdfs://localhost:41555/backupUT/backup_1733316442145/backup/system_bulk/ 2024-12-04T12:47:37,406 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-04T12:47:37,437 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:41555, inputRoot=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:47:37,437 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_430596070_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:41555/backupUT/backup_1733316442145/backup/system_bulk, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:41555/backupUT/backup_1733316442145/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1733316446556_backup_system_bulk 2024-12-04T12:47:37,439 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-04T12:47:37,443 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_1733316446556_backup_system_bulk to hdfs://localhost:41555/backupUT/backup_1733316442145/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1733316446556_backup_system_bulk 2024-12-04T12:47:37,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741931_1107 (size=702) 2024-12-04T12:47:37,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741932_1108 (size=76) 2024-12-04T12:47:37,459 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:37,459 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:37,459 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:38,212 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-591222896875845159.jar 2024-12-04T12:47:38,213 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:38,213 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:38,213 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:38,214 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:38,214 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:38,214 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:38,214 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:38,214 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:38,215 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:38,215 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:47:38,215 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T12:47:38,215 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T12:47:38,216 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T12:47:38,216 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T12:47:38,216 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T12:47:38,216 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T12:47:38,216 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T12:47:38,216 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T12:47:38,217 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T12:47:38,217 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T12:47:38,217 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:38,217 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:38,218 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:38,218 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:38,218 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:38,218 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:38,218 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:38,253 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:47:38,276 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733316446556_backup_system_bulk' hfile list 2024-12-04T12:47:38,353 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local146479308_0005_defc0926-e4f7-4ba7-93f9-bbf67aa95ddf/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T12:47:38,353 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local146479308_0005_defc0926-e4f7-4ba7-93f9-bbf67aa95ddf/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T12:47:39,392 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-04T12:47:39,395 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-04T12:47:39,405 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733316446556_backup_system_bulk 2024-12-04T12:47:39,405 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733316446556_backup_system_bulk finished. 2024-12-04T12:47:39,405 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733316442145_ns3:test-17333163965972 2024-12-04T12:47:39,405 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733316448726_ns3_test-17333163965972 to hdfs://localhost:41555/backupUT/backup_1733316442145/ns3/test-17333163965972/ 2024-12-04T12:47:39,405 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-04T12:47:39,440 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:41555, inputRoot=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:47:39,440 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_430596070_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:41555/backupUT/backup_1733316442145/ns3/test-17333163965972, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:41555/backupUT/backup_1733316442145/ns3/test-17333163965972/.hbase-snapshot/.tmp/snapshot_1733316448726_ns3_test-17333163965972 2024-12-04T12:47:39,442 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-04T12:47:39,446 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_1733316448726_ns3_test-17333163965972 to hdfs://localhost:41555/backupUT/backup_1733316442145/ns3/test-17333163965972/.hbase-snapshot/.tmp/snapshot_1733316448726_ns3_test-17333163965972 2024-12-04T12:47:39,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741933_1109 (size=86) 2024-12-04T12:47:39,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741934_1110 (size=443) 2024-12-04T12:47:39,858 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:39,858 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:39,858 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:40,612 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-5571899699474127923.jar 2024-12-04T12:47:40,612 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:40,613 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:40,613 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:40,613 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:40,613 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:40,613 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:40,614 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:40,614 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:40,614 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:40,614 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:47:40,614 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T12:47:40,614 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T12:47:40,615 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T12:47:40,615 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T12:47:40,615 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T12:47:40,615 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T12:47:40,615 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T12:47:40,615 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T12:47:40,616 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T12:47:40,616 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T12:47:40,616 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:40,616 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:40,616 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:40,617 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:40,617 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:40,617 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:40,617 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:40,649 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:47:40,672 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733316448726_ns3_test-17333163965972' hfile list 2024-12-04T12:47:40,760 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local537798544_0006_14d108e0-e124-4731-9cff-7e636c2b2e93/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T12:47:40,760 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local537798544_0006_14d108e0-e124-4731-9cff-7e636c2b2e93/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T12:47:41,798 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-04T12:47:41,800 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-04T12:47:41,811 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733316448726_ns3_test-17333163965972 2024-12-04T12:47:41,811 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733316448726_ns3_test-17333163965972 finished. 2024-12-04T12:47:41,811 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733316442145_test-1733316396597 2024-12-04T12:47:41,811 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733316450904_default_test-1733316396597 to hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/ 2024-12-04T12:47:41,811 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-04T12:47:41,838 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:41555, inputRoot=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:47:41,838 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_430596070_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/.hbase-snapshot/.tmp/snapshot_1733316450904_default_test-1733316396597 2024-12-04T12:47:41,840 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-04T12:47:41,846 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_1733316450904_default_test-1733316396597 to hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/.hbase-snapshot/.tmp/snapshot_1733316450904_default_test-1733316396597 2024-12-04T12:47:41,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741935_1111 (size=84) 2024-12-04T12:47:41,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741936_1112 (size=1347) 2024-12-04T12:47:42,259 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:42,260 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:42,260 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:43,021 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-3140772731607697027.jar 2024-12-04T12:47:43,022 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:43,022 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:43,022 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:43,023 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:43,023 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:43,023 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:43,023 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:43,024 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:43,024 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:43,024 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:47:43,024 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T12:47:43,024 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T12:47:43,025 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T12:47:43,025 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T12:47:43,025 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T12:47:43,025 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T12:47:43,026 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T12:47:43,026 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T12:47:43,026 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T12:47:43,026 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T12:47:43,027 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:43,027 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:43,027 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:43,027 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:43,027 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:43,028 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:43,028 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:43,063 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:47:43,086 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733316450904_default_test-1733316396597' hfile list 2024-12-04T12:47:43,088 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=43.0 K 2024-12-04T12:47:43,162 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local774307650_0007_d964833f-809c-4407-80a3-d100b3efd9b8/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T12:47:43,162 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local774307650_0007_d964833f-809c-4407-80a3-d100b3efd9b8/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T12:47:43,225 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-04T12:47:43,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741937_1113 (size=13760) 2024-12-04T12:47:43,661 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/5799223a4841435fb10d4958b6b73deb output=hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/5799223a4841435fb10d4958b6b73deb 2024-12-04T12:47:43,661 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=13760 (13.4 K) time=0sec 0.030M/sec 2024-12-04T12:47:43,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741938_1114 (size=10796) 2024-12-04T12:47:44,091 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/ed2773986acc4ae18188f34dec668da1 output=hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/ed2773986acc4ae18188f34dec668da1 2024-12-04T12:47:44,092 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=10796 (10.5 K) time=0sec 0.024M/sec 2024-12-04T12:47:44,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741939_1115 (size=8413) 2024-12-04T12:47:44,530 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40 output=hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40 2024-12-04T12:47:44,530 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.019M/sec 2024-12-04T12:47:44,543 WARN [LocalJobRunner Map Task Executor #0 {}] mapred.Task(640): Task status: "copied 5.5 K/5.5 K (100.0%) from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/mobdir/data/default/test-1733316396597/64682aa8ca2d45d6ee9ace61337cabc4/mob/d41d8cd98f00b204e9800998ecf8427e20241204f4e86055bf4b405ab07a7b9c2fae1c0b_87e3e36a673c628af1440164c1fd3443 to hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/archive/data/default/test-1733316396597/64682aa8ca2d45d6ee9ace61337cabc4/mob/d41d8cd98f00b204e9800998ecf8427e20241204f4e86055bf4b405ab07a7b9c2fae1c0b_87e3e36a673c628af1440164c1fd3443" truncated to max limit (512 characters) 2024-12-04T12:47:44,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741940_1116 (size=5591) 2024-12-04T12:47:44,959 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/mobdir/data/default/test-1733316396597/64682aa8ca2d45d6ee9ace61337cabc4/mob/d41d8cd98f00b204e9800998ecf8427e20241204f4e86055bf4b405ab07a7b9c2fae1c0b_87e3e36a673c628af1440164c1fd3443 output=hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/archive/data/default/test-1733316396597/64682aa8ca2d45d6ee9ace61337cabc4/mob/d41d8cd98f00b204e9800998ecf8427e20241204f4e86055bf4b405ab07a7b9c2fae1c0b_87e3e36a673c628af1440164c1fd3443 2024-12-04T12:47:44,960 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5591 (5.5 K) time=0sec 0.013M/sec 2024-12-04T12:47:44,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741941_1117 (size=5484) 2024-12-04T12:47:45,395 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/6759befaa3d44e67a2be163409a2ab50 output=hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/6759befaa3d44e67a2be163409a2ab50 2024-12-04T12:47:45,395 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-12-04T12:47:46,200 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-04T12:47:46,201 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-04T12:47:46,212 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733316450904_default_test-1733316396597 2024-12-04T12:47:46,213 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733316450904_default_test-1733316396597 finished. 2024-12-04T12:47:46,213 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733316442145_ns4:test-17333163965973 2024-12-04T12:47:46,213 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733316453075_ns4_test-17333163965973 to hdfs://localhost:41555/backupUT/backup_1733316442145/ns4/test-17333163965973/ 2024-12-04T12:47:46,213 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-04T12:47:46,241 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:41555, inputRoot=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:47:46,241 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_430596070_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:41555/backupUT/backup_1733316442145/ns4/test-17333163965973, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:41555/backupUT/backup_1733316442145/ns4/test-17333163965973/.hbase-snapshot/.tmp/snapshot_1733316453075_ns4_test-17333163965973 2024-12-04T12:47:46,243 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-04T12:47:46,247 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_1733316453075_ns4_test-17333163965973 to hdfs://localhost:41555/backupUT/backup_1733316442145/ns4/test-17333163965973/.hbase-snapshot/.tmp/snapshot_1733316453075_ns4_test-17333163965973 2024-12-04T12:47:46,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741943_1119 (size=86) 2024-12-04T12:47:46,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741942_1118 (size=443) 2024-12-04T12:47:46,662 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:46,663 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:46,664 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:47,421 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-9845304245366655999.jar 2024-12-04T12:47:47,421 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:47,422 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:47,422 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:47,422 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:47,422 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:47,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:47,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:47,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:47,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:47,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:47:47,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T12:47:47,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T12:47:47,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T12:47:47,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T12:47:47,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T12:47:47,425 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T12:47:47,425 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T12:47:47,425 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T12:47:47,425 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T12:47:47,425 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T12:47:47,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:47,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:47,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:47,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:47,427 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:47,427 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:47,427 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:47,461 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:47:47,487 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733316453075_ns4_test-17333163965973' hfile list 2024-12-04T12:47:47,568 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local1253150413_0008_f8aedbaf-8bb8-4ffd-907a-ed8f8dde7d2e/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T12:47:47,568 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local1253150413_0008_f8aedbaf-8bb8-4ffd-907a-ed8f8dde7d2e/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T12:47:48,606 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-04T12:47:48,608 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-04T12:47:48,617 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733316453075_ns4_test-17333163965973 2024-12-04T12:47:48,617 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733316453075_ns4_test-17333163965973 finished. 2024-12-04T12:47:48,617 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733316442145_ns2:test-17333163965971 2024-12-04T12:47:48,617 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733316455236_ns2_test-17333163965971 to hdfs://localhost:41555/backupUT/backup_1733316442145/ns2/test-17333163965971/ 2024-12-04T12:47:48,617 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-04T12:47:48,642 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:41555, inputRoot=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:47:48,642 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_430596070_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:41555/backupUT/backup_1733316442145/ns2/test-17333163965971, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:41555/backupUT/backup_1733316442145/ns2/test-17333163965971/.hbase-snapshot/.tmp/snapshot_1733316455236_ns2_test-17333163965971 2024-12-04T12:47:48,644 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-04T12:47:48,648 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_1733316455236_ns2_test-17333163965971 to hdfs://localhost:41555/backupUT/backup_1733316442145/ns2/test-17333163965971/.hbase-snapshot/.tmp/snapshot_1733316455236_ns2_test-17333163965971 2024-12-04T12:47:48,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741944_1120 (size=86) 2024-12-04T12:47:48,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741945_1121 (size=496) 2024-12-04T12:47:49,066 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:49,067 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:49,068 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:49,818 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-5533097494776514027.jar 2024-12-04T12:47:49,819 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:49,819 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:49,819 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:49,819 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:49,820 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:49,820 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:49,820 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:49,820 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:49,821 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:47:49,821 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:47:49,821 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T12:47:49,821 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T12:47:49,821 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T12:47:49,822 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T12:47:49,822 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T12:47:49,822 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T12:47:49,822 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T12:47:49,822 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T12:47:49,822 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T12:47:49,823 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T12:47:49,823 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:49,823 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:49,823 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:49,824 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:49,824 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T12:47:49,824 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:49,824 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:47:49,856 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:47:49,882 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733316455236_ns2_test-17333163965971' hfile list 2024-12-04T12:47:49,885 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=13.2 K 2024-12-04T12:47:49,971 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local274032691_0009_7cc9a1f7-a81c-4b9a-b648-924524197c4f/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T12:47:49,971 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local274032691_0009_7cc9a1f7-a81c-4b9a-b648-924524197c4f/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T12:47:50,035 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-04T12:47:50,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741946_1122 (size=8413) 2024-12-04T12:47:50,455 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/194e38d3f9c54ed88dcc2848906ca7b4 output=hdfs://localhost:41555/backupUT/backup_1733316442145/ns2/test-17333163965971/archive/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/194e38d3f9c54ed88dcc2848906ca7b4 2024-12-04T12:47:50,455 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.019M/sec 2024-12-04T12:47:50,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741947_1123 (size=5134) 2024-12-04T12:47:50,891 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/9fe180fa04404396b0429daa9427c60f output=hdfs://localhost:41555/backupUT/backup_1733316442145/ns2/test-17333163965971/archive/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/9fe180fa04404396b0429daa9427c60f 2024-12-04T12:47:50,891 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5134 (5.0 K) time=0sec 0.011M/sec 2024-12-04T12:47:51,013 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-04T12:47:51,014 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-04T12:47:51,022 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733316455236_ns2_test-17333163965971 2024-12-04T12:47:51,022 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733316455236_ns2_test-17333163965971 finished. 2024-12-04T12:47:51,022 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): backup:system_bulk 2024-12-04T12:47:51,022 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns3:test-17333163965972 2024-12-04T12:47:51,022 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1733316396597 2024-12-04T12:47:51,022 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns4:test-17333163965973 2024-12-04T12:47:51,022 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17333163965971 2024-12-04T12:47:51,033 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733316442145 2024-12-04T12:47:51,033 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-12-04T12:47:51,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741948_1124 (size=176) 2024-12-04T12:47:51,440 INFO [Time-limited test {}] impl.BackupManifest(504): Manifest file stored to hdfs://localhost:41555/backupUT/backup_1733316442145/.backup.manifest 2024-12-04T12:47:51,440 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1733316442145 finished: type=FULL,tablelist=backup:system_bulk;ns3:test-17333163965972;test-1733316396597;ns4:test-17333163965973;ns2:test-17333163965971,targetRootDir=hdfs://localhost:41555/backupUT,startts=1733316446335,completets=1733316471033,bytescopied=0 2024-12-04T12:47:51,440 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-12-04T12:47:51,440 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733316446556_backup_system_bulk 2024-12-04T12:47:51,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733316446556_backup_system_bulk" type: DISABLED 2024-12-04T12:47:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733316446556_backup_system_bulk 2024-12-04T12:47:51,451 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733316446556_backup_system_bulk for backup backup_1733316442145 succeeded. 2024-12-04T12:47:51,451 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733316448726_ns3_test-17333163965972 2024-12-04T12:47:51,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733316448726_ns3_test-17333163965972" type: DISABLED 2024-12-04T12:47:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733316448726_ns3_test-17333163965972 2024-12-04T12:47:51,456 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733316448726_ns3_test-17333163965972 for backup backup_1733316442145 succeeded. 2024-12-04T12:47:51,456 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733316450904_default_test-1733316396597 2024-12-04T12:47:51,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733316450904_default_test-1733316396597" type: DISABLED 2024-12-04T12:47:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733316450904_default_test-1733316396597 2024-12-04T12:47:51,461 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733316450904_default_test-1733316396597 for backup backup_1733316442145 succeeded. 2024-12-04T12:47:51,461 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733316453075_ns4_test-17333163965973 2024-12-04T12:47:51,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733316453075_ns4_test-17333163965973" type: DISABLED 2024-12-04T12:47:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733316453075_ns4_test-17333163965973 2024-12-04T12:47:51,466 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733316453075_ns4_test-17333163965973 for backup backup_1733316442145 succeeded. 2024-12-04T12:47:51,466 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733316455236_ns2_test-17333163965971 2024-12-04T12:47:51,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733316455236_ns2_test-17333163965971" type: DISABLED 2024-12-04T12:47:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733316455236_ns2_test-17333163965971 2024-12-04T12:47:51,471 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733316455236_ns2_test-17333163965971 for backup backup_1733316442145 succeeded. 2024-12-04T12:47:51,473 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-04T12:47:51,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-12-04T12:47:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-04T12:47:51,484 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-04T12:47:51,488 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-04T12:47:51,493 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1733316442145 completed. 2024-12-04T12:47:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:51,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:55,583 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T12:47:56,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:56,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:56,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:56,571 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:56,573 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-04T12:47:56,577 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733316476524 2024-12-04T12:47:56,580 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733316442145} 2024-12-04T12:47:56,580 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-12-04T12:47:56,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:56,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:56,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:56,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:56,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:56,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:56,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:56,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:56,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:47:56,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:47:56,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T12:47:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733316476681 (current time:1733316476681). 2024-12-04T12:47:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T12:47:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-04T12:47:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ddb89c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:47:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:47:56,683 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:47:56,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:47:56,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:47:56,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41959424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:56,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:47:56,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:47:56,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:56,685 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:47:56,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3240ff23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:47:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:47:56,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:47:56,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:47:56,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:47:56,690 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049. 2024-12-04T12:47:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:47:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:47:56,691 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:47:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T12:47:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T12:47:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-12-04T12:47:56,694 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T12:47:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T12:47:56,695 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T12:47:56,698 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T12:47:56,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741949_1125 (size=52) 2024-12-04T12:47:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T12:47:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T12:47:57,106 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T12:47:57,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949}] 2024-12-04T12:47:57,108 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:47:57,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38879 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2024-12-04T12:47:57,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:47:57,262 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 9dab1d763bdadc4adf3b254f1a913949 2/2 column families, dataSize=2.54 KB heapSize=4.22 KB 2024-12-04T12:47:57,273 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/meta/49f7a0d1031c46f081313e8d0efdd5a4 is 158, key is trslm:hdfs://localhost:41555/backupUT\x00ns2:test-17333163965971/meta:log-roll-map/1733316471027/Put/seqid=0 2024-12-04T12:47:57,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741950_1126 (size=6646) 2024-12-04T12:47:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T12:47:57,680 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/meta/49f7a0d1031c46f081313e8d0efdd5a4 2024-12-04T12:47:57,695 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/session/d83b2598c8bb4a20a3c649488d12f879 is 826, key is session:backup_1733316442145/session:context/1733316471486/Put/seqid=0 2024-12-04T12:47:57,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741951_1127 (size=5928) 2024-12-04T12:47:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T12:47:58,102 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/session/d83b2598c8bb4a20a3c649488d12f879 2024-12-04T12:47:58,116 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/meta/49f7a0d1031c46f081313e8d0efdd5a4 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/49f7a0d1031c46f081313e8d0efdd5a4 2024-12-04T12:47:58,125 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/49f7a0d1031c46f081313e8d0efdd5a4, entries=12, sequenceid=42, filesize=6.5 K 2024-12-04T12:47:58,127 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/session/d83b2598c8bb4a20a3c649488d12f879 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/d83b2598c8bb4a20a3c649488d12f879 2024-12-04T12:47:58,135 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/d83b2598c8bb4a20a3c649488d12f879, entries=2, sequenceid=42, filesize=5.8 K 2024-12-04T12:47:58,136 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~2.54 KB/2604, heapSize ~4.19 KB/4288, currentSize=0 B/0 for 9dab1d763bdadc4adf3b254f1a913949 in 875ms, sequenceid=42, compaction requested=true 2024-12-04T12:47:58,136 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 9dab1d763bdadc4adf3b254f1a913949: 2024-12-04T12:47:58,136 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. for snapshot_backup_system completed. 2024-12-04T12:47:58,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949.' region-info for snapshot=snapshot_backup_system 2024-12-04T12:47:58,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T12:47:58,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/6bb6c66690d84dfb8bc5bd7b77d4a104, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/bb0f881d4a4a457486fe958ca934bea8, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/49f7a0d1031c46f081313e8d0efdd5a4] hfiles 2024-12-04T12:47:58,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/6bb6c66690d84dfb8bc5bd7b77d4a104 for snapshot=snapshot_backup_system 2024-12-04T12:47:58,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/bb0f881d4a4a457486fe958ca934bea8 for snapshot=snapshot_backup_system 2024-12-04T12:47:58,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/49f7a0d1031c46f081313e8d0efdd5a4 for snapshot=snapshot_backup_system 2024-12-04T12:47:58,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/0a06d46ae887482d95f1e19410656caa, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/5d1327784fd34e43a68805beada718ac, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/156393e147e744b0aa829f60f1c74c5e, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/d83b2598c8bb4a20a3c649488d12f879] hfiles 2024-12-04T12:47:58,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/4): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/0a06d46ae887482d95f1e19410656caa for snapshot=snapshot_backup_system 2024-12-04T12:47:58,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/4): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/5d1327784fd34e43a68805beada718ac for snapshot=snapshot_backup_system 2024-12-04T12:47:58,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/4): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/156393e147e744b0aa829f60f1c74c5e for snapshot=snapshot_backup_system 2024-12-04T12:47:58,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (4/4): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/d83b2598c8bb4a20a3c649488d12f879 for snapshot=snapshot_backup_system 2024-12-04T12:47:58,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741952_1128 (size=330) 2024-12-04T12:47:58,544 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:47:58,544 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-04T12:47:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-04T12:47:58,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:47:58,546 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:47:58,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-04T12:47:58,551 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T12:47:58,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 9dab1d763bdadc4adf3b254f1a913949 in 1.4410 sec 2024-12-04T12:47:58,552 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T12:47:58,554 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T12:47:58,554 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-04T12:47:58,555 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-04T12:47:58,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741953_1129 (size=967) 2024-12-04T12:47:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T12:47:58,975 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T12:47:58,987 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T12:47:58,987 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.hbase-snapshot/snapshot_backup_system 2024-12-04T12:47:58,989 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T12:47:58,989 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-12-04T12:47:58,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2980 sec 2024-12-04T12:48:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T12:48:00,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-04T12:48:00,865 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733316476524 started at 1733316480865. 2024-12-04T12:48:00,870 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733316476524 has been started. 2024-12-04T12:48:00,877 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [backup:system_bulk, ns2:test-17333163965971, ns3:test-17333163965972, ns4:test-17333163965973, test-1733316396597] 2024-12-04T12:48:00,882 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1733316432958for backupID backup_1733316476524 2024-12-04T12:48:00,882 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-12-04T12:48:00,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-12-04T12:48:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-04T12:48:00,883 INFO [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-04T12:48:00,883 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-04T12:48:00,883 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-04T12:48:00,883 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-04T12:48:00,884 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T12:48:00,884 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T12:48:00,961 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:48:00,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T12:48:00,961 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T12:48:00,961 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T12:48:00,962 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:48:00,962 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-04T12:48:00,962 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-04T12:48:00,962 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T12:48:00,963 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-04T12:48:00,963 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T12:48:00,963 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-04T12:48:00,963 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-04T12:48:00,963 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-04T12:48:00,963 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-04T12:48:00,963 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-04T12:48:00,963 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-04T12:48:00,963 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-04T12:48:00,963 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '938b79ed23df,38879,1733316387992' joining acquired barrier for procedure (rolllog) in zk 2024-12-04T12:48:00,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:48:00,980 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-04T12:48:00,980 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:48:00,980 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T12:48:00,980 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T12:48:00,980 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-04T12:48:00,980 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-04T12:48:00,980 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T12:48:00,981 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T12:48:00,981 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:48:00,981 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:48:00,982 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T12:48:00,982 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '938b79ed23df,38879,1733316387992' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-04T12:48:00,982 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@704f4346[Count = 0] remaining members to acquire global barrier 2024-12-04T12:48:00,982 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-04T12:48:00,982 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-04T12:48:00,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T12:48:00,989 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-04T12:48:00,989 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-04T12:48:00,989 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-04T12:48:00,989 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:48:00,989 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-04T12:48:00,989 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-04T12:48:00,989 DEBUG [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 938b79ed23df,38879,1733316387992 2024-12-04T12:48:00,989 INFO [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733316446425 highest: 1733316446461 on 938b79ed23df,38879,1733316387992 2024-12-04T12:48:00,989 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0:(num 1733316446425) roll requested 2024-12-04T12:48:01,001 DEBUG [regionserver/938b79ed23df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316480990, exclude list is [], retry=0 2024-12-04T12:48:01,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:48:01,005 INFO [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316446425 with entries=2, filesize=645 B; new WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316480990 2024-12-04T12:48:01,006 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:48:01,006 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316446425 is not closed yet, will try archiving it next time 2024-12-04T12:48:01,006 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 938b79ed23df%2C38879%2C1733316387992.meta:.meta(num 1733316446445) roll requested 2024-12-04T12:48:01,006 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316413592 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316413592 2024-12-04T12:48:01,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741911_1087 (size=653) 2024-12-04T12:48:01,007 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316446425 not finished, retry = 0 2024-12-04T12:48:01,017 DEBUG [regionserver/938b79ed23df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316481006.meta, exclude list is [], retry=0 2024-12-04T12:48:01,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:48:01,020 INFO [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316446445.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316481006.meta 2024-12-04T12:48:01,020 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:48:01,020 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316446445.meta is not closed yet, will try archiving it next time 2024-12-04T12:48:01,020 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1:(num 1733316446461) roll requested 2024-12-04T12:48:01,022 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316446445.meta not finished, retry = 0 2024-12-04T12:48:01,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741912_1088 (size=93) 2024-12-04T12:48:01,031 DEBUG [regionserver/938b79ed23df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316481020, exclude list is [], retry=0 2024-12-04T12:48:01,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44611,DS-fb7dda59-85b1-4c17-b284-5c4de7371519,DISK] 2024-12-04T12:48:01,035 INFO [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316446461 with entries=17, filesize=4.59 KB; new WAL /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316481020 2024-12-04T12:48:01,035 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45053:45053)] 2024-12-04T12:48:01,035 DEBUG [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316446461 is not closed yet, will try archiving it next time 2024-12-04T12:48:01,035 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316432958 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316432958 2024-12-04T12:48:01,036 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316446461 not finished, retry = 0 2024-12-04T12:48:01,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741913_1089 (size=4710) 2024-12-04T12:48:01,050 DEBUG [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-12-04T12:48:01,050 INFO [rs(938b79ed23df,38879,1733316387992)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733316480990 on 938b79ed23df,38879,1733316387992 2024-12-04T12:48:01,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:01,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:01,071 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-04T12:48:01,071 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '938b79ed23df,38879,1733316387992' in zk 2024-12-04T12:48:01,080 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-04T12:48:01,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:48:01,080 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T12:48:01,080 DEBUG [member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-04T12:48:01,080 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:48:01,080 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T12:48:01,080 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T12:48:01,081 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T12:48:01,081 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T12:48:01,082 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:48:01,082 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:48:01,082 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T12:48:01,083 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:48:01,083 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:48:01,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '938b79ed23df,38879,1733316387992': 2024-12-04T12:48:01,084 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '938b79ed23df,38879,1733316387992' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-04T12:48:01,084 INFO [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-04T12:48:01,084 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-04T12:48:01,084 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-04T12:48:01,085 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-04T12:48:01,085 INFO [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-04T12:48:01,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T12:48:01,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T12:48:01,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T12:48:01,096 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-04T12:48:01,096 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-04T12:48:01,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T12:48:01,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T12:48:01,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T12:48:01,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T12:48:01,097 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T12:48:01,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T12:48:01,098 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T12:48:01,098 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:48:01,098 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:48:01,099 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T12:48:01,099 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:48:01,100 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:48:01,100 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:48:01,101 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T12:48:01,101 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T12:48:01,102 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------938b79ed23df,38879,1733316387992 2024-12-04T12:48:01,108 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316446425 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316446425 2024-12-04T12:48:01,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:48:01,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T12:48:01,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T12:48:01,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T12:48:01,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T12:48:01,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/938b79ed23df,38879,1733316387992 2024-12-04T12:48:01,114 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T12:48:01,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T12:48:01,114 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T12:48:01,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T12:48:01,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T12:48:01,114 DEBUG [(938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T12:48:01,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-04T12:48:01,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-04T12:48:01,114 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T12:48:01,114 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T12:48:01,123 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316446445.meta to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.meta.1733316446445.meta 2024-12-04T12:48:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-04T12:48:01,126 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {938b79ed23df:38879=1733316432958} newestTimestamps: {938b79ed23df:38879=1733316446461} 2024-12-04T12:48:01,127 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 938b79ed23df%2C38879%2C1733316387992.meta.1733316392177.meta 2024-12-04T12:48:01,128 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 938b79ed23df%2C38879%2C1733316387992.meta.1733316432943.meta 2024-12-04T12:48:01,128 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 938b79ed23df%2C38879%2C1733316387992.meta.1733316481006.meta 2024-12-04T12:48:01,128 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316480990 2024-12-04T12:48:01,128 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316446461 2024-12-04T12:48:01,128 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316481020 2024-12-04T12:48:01,129 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.meta.1733316413617.meta 2024-12-04T12:48:01,129 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.meta.1733316446445.meta 2024-12-04T12:48:01,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741957_1133 (size=640) 2024-12-04T12:48:01,543 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:backup:system_bulk target: hdfs://localhost:41555/backupUT/backup_1733316476524/backup/system_bulk descriptor: 'backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:48:01,543 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-04T12:48:01,550 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table backup:system_bulk 2024-12-04T12:48:01,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741958_1134 (size=45) 2024-12-04T12:48:01,957 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table backup:system_bulk 2024-12-04T12:48:01,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741959_1135 (size=390) 2024-12-04T12:48:02,373 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns3:test-17333163965972 target: hdfs://localhost:41555/backupUT/backup_1733316476524/ns3/test-17333163965972 descriptor: 'ns3:test-17333163965972', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:48:02,373 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-04T12:48:02,379 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns3:test-17333163965972 2024-12-04T12:48:02,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741960_1136 (size=50) 2024-12-04T12:48:02,788 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns3:test-17333163965972 2024-12-04T12:48:02,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741961_1137 (size=950) 2024-12-04T12:48:03,210 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1733316396597 target: hdfs://localhost:41555/backupUT/backup_1733316476524/default/test-1733316396597 descriptor: 'test-1733316396597', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:48:03,210 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-04T12:48:03,216 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1733316396597 2024-12-04T12:48:03,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741962_1138 (size=53) 2024-12-04T12:48:03,622 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1733316396597 2024-12-04T12:48:03,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741963_1139 (size=390) 2024-12-04T12:48:04,040 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns4:test-17333163965973 target: hdfs://localhost:41555/backupUT/backup_1733316476524/ns4/test-17333163965973 descriptor: 'ns4:test-17333163965973', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:48:04,040 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-04T12:48:04,045 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns4:test-17333163965973 2024-12-04T12:48:04,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741964_1140 (size=50) 2024-12-04T12:48:04,453 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns4:test-17333163965973 2024-12-04T12:48:04,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741965_1141 (size=364) 2024-12-04T12:48:04,872 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17333163965971 target: hdfs://localhost:41555/backupUT/backup_1733316476524/ns2/test-17333163965971 descriptor: 'ns2:test-17333163965971', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:48:04,873 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-04T12:48:04,878 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17333163965971 2024-12-04T12:48:04,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741966_1142 (size=50) 2024-12-04T12:48:05,287 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17333163965971 2024-12-04T12:48:05,303 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:41555/backupUT/.tmp/backup_1733316476524 from hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316446461;hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316446425 2024-12-04T12:48:05,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e72e204, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:05,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:05,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:05,304 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:05,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:05,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:05,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb0bf0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:05,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:05,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:05,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:05,305 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:05,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38529ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:05,306 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:05,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:05,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@58417403 2024-12-04T12:48:05,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:05,308 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:05,311 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-04T12:48:05,312 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table backup:system_bulk 2024-12-04T12:48:05,312 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:05,312 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:05,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:05,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:05,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:05,316 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for backup:system_bulk: backup:system_bulk; 2024-12-04T12:48:05,316 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17333163965971 2024-12-04T12:48:05,316 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17333163965971,, stopping at row=ns2:test-17333163965971 ,, for max=2147483647 with caching=100 2024-12-04T12:48:05,318 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17333163965971: ns2:test-17333163965971; 2024-12-04T12:48:05,318 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns3:test-17333163965972 2024-12-04T12:48:05,318 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns3:test-17333163965972,, stopping at row=ns3:test-17333163965972 ,, for max=2147483647 with caching=100 2024-12-04T12:48:05,320 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns3:test-17333163965972: ns3:test-17333163965972; 2024-12-04T12:48:05,320 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns4:test-17333163965973 2024-12-04T12:48:05,320 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns4:test-17333163965973,, stopping at row=ns4:test-17333163965973 ,, for max=2147483647 with caching=100 2024-12-04T12:48:05,322 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns4:test-17333163965973: ns4:test-17333163965973; 2024-12-04T12:48:05,322 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1733316396597 2024-12-04T12:48:05,322 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733316396597,, stopping at row=test-1733316396597 ,, for max=2147483647 with caching=100 2024-12-04T12:48:05,324 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1733316396597: test-1733316396597; 2024-12-04T12:48:05,324 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 5 reduce partitions to match current region count for all tables 2024-12-04T12:48:05,324 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/partitions_42a686f0-156d-47ab-bd72-865a60c29711 2024-12-04T12:48:05,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741967_1143 (size=324) 2024-12-04T12:48:05,734 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:05,735 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:05,736 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,489 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-7278030160452438356.jar 2024-12-04T12:48:06,489 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,489 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,489 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,490 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,490 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,490 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,490 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,490 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,490 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,491 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:48:06,491 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T12:48:06,491 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T12:48:06,491 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T12:48:06,491 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T12:48:06,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T12:48:06,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T12:48:06,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T12:48:06,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T12:48:06,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T12:48:06,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T12:48:06,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:06,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:48:06,494 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: backup:system_bulk,ns2:test-17333163965971,ns3:test-17333163965972,ns4:test-17333163965973,default:test-1733316396597 2024-12-04T12:48:06,494 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:48:06,494 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:188) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:06,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:06,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:06,494 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:06,494 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:48:06,973 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 87e3e36a673c628af1440164c1fd3443, had cached 0 bytes from a total of 38453 2024-12-04T12:48:07,242 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-8176138491585682718.jar 2024-12-04T12:48:07,251 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:48:07,275 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316446461 for WAL files 2024-12-04T12:48:07,276 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316446461 2024-12-04T12:48:07,276 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316446425 for WAL files 2024-12-04T12:48:07,278 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316446425 2024-12-04T12:48:07,348 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local426735631_0010_9d88bf61-ddc2-4354-aba5-de2d34d36790/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T12:48:07,348 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local426735631_0010_9d88bf61-ddc2-4354-aba5-de2d34d36790/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T12:48:07,396 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316446461 for hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316446461 (-9223372036854775808:9223372036854775807) length:4710 2024-12-04T12:48:07,400 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-12-04T12:48:07,400 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-12-04T12:48:07,432 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316446425 for hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0.1733316446425 (-9223372036854775808:9223372036854775807) length:653 2024-12-04T12:48:07,434 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-12-04T12:48:07,434 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-12-04T12:48:07,514 DEBUG [pool-260-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59600a0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:07,514 DEBUG [pool-260-thread-1 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:07,515 DEBUG [pool-260-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:07,516 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:07,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:07,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:07,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c65e7ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:07,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:07,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:07,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:07,518 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:07,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f2335dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:07,519 DEBUG [pool-260-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:07,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:07,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:07,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:07,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733316396597', row='row-test-1733316396597-2-0', locateType=CURRENT is [region=test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443., hostname=938b79ed23df,38879,1733316387992, seqNum=338] 2024-12-04T12:48:07,523 INFO [pool-260-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-260-thread-1. 2024-12-04T12:48:07,524 DEBUG [pool-260-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:07,524 DEBUG [pool-260-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:07,524 DEBUG [pool-260-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:07,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:07,524 DEBUG [pool-260-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733316396597-2-0] 2024-12-04T12:48:07,524 DEBUG [pool-260-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 938b79ed23df 2024-12-04T12:48:07,528 DEBUG [pool-260-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/backupUT/.tmp/backup_1733316476524/_temporary/0/_temporary/attempt_local426735631_0010_r_000004_0/default/test-1733316396597/f2/4922ab9176074c7f84f86d1e2869075e is 58, key is row-test-1733316396597-2-0/f2:q1/1733316471514/Put/seqid=0 2024-12-04T12:48:07,530 WARN [Thread-1516 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [938b79ed23df:38879] Specified favored nodes: [938b79ed23df:38879] 2024-12-04T12:48:07,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741968_1144 (size=5366) 2024-12-04T12:48:08,391 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:41555/backupUT 2024-12-04T12:48:08,391 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1733316476524 2024-12-04T12:48:08,392 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-12-04T12:48:08,403 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:41555/backupUT/.tmp/backup_1733316476524, hdfs://localhost:41555/backupUT] 2024-12-04T12:48:08,452 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:48:08,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741969_1145 (size=5366) 2024-12-04T12:48:09,054 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-12-04T12:48:09,057 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1733316476524 2024-12-04T12:48:09,057 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 5366 bytes copied." 2024-12-04T12:48:09,057 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local1032043958_0011 completed: true true 2024-12-04T12:48:09,058 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=6688400 FILE: Number of bytes written=17209939 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=410795 HDFS: Number of bytes written=2548192 HDFS: Number of read operations=1171 HDFS: Number of large read operations=0 HDFS: Number of write operations=422 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=1482 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=5366 Bytes Copied=5366 Bytes Expected=5366 Files Copied=2 DIR_COPY=4 2024-12-04T12:48:09,058 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:41555/backupUT/.tmp/backup_1733316476524 to hdfs://localhost:41555/backupUT finished. 2024-12-04T12:48:09,077 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733316476524 2024-12-04T12:48:09,081 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733316442145} 2024-12-04T12:48:09,081 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-12-04T12:48:09,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741970_1146 (size=648) 2024-12-04T12:48:09,488 INFO [Time-limited test {}] impl.BackupManifest(504): Manifest file stored to hdfs://localhost:41555/backupUT/backup_1733316476524/.backup.manifest 2024-12-04T12:48:09,489 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1733316476524 finished: type=INCREMENTAL,tablelist=backup:system_bulk;ns3:test-17333163965972;test-1733316396597;ns4:test-17333163965973;ns2:test-17333163965971,targetRootDir=hdfs://localhost:41555/backupUT,startts=1733316480865,completets=1733316489077,bytescopied=0 2024-12-04T12:48:09,489 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-04T12:48:09,503 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-12-04T12:48:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-04T12:48:09,507 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-04T12:48:09,510 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-04T12:48:09,515 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1733316476524 completed. 2024-12-04T12:48:09,515 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-12-04T12:48:09,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:09,529 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:41555/backupUT/backup_1733316476524/.backup.manifest 2024-12-04T12:48:09,531 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733316476524/.backup.manifest 2024-12-04T12:48:09,532 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(196): Restoring full backup_1733316406744 2024-12-04T12:48:09,535 INFO [Time-limited test {}] impl.RestoreTablesClient(100): HBase table table1 does not exist. It will be created during restore process 2024-12-04T12:48:09,536 INFO [Time-limited test {}] impl.RestoreTablesClient(100): HBase table ns2:table2 does not exist. It will be created during restore process 2024-12-04T12:48:09,537 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:41555/backupUT/backup_1733316406744/.backup.manifest 2024-12-04T12:48:09,539 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733316406744/.backup.manifest 2024-12-04T12:48:09,541 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:41555/backupUT/backup_1733316406744/.backup.manifest 2024-12-04T12:48:09,543 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733316406744/.backup.manifest 2024-12-04T12:48:09,543 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'test-1733316396597' to 'table1' from full backup image hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597 2024-12-04T12:48:09,550 INFO [Time-limited test {}] util.RestoreTool(491): Creating target table 'table1' 2024-12-04T12:48:09,551 DEBUG [Time-limited test {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443 2024-12-04T12:48:09,551 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f in region [hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443] 2024-12-04T12:48:09,555 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40 first=row0 last=row98 2024-12-04T12:48:09,555 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f3 in region [hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443] 2024-12-04T12:48:09,559 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f3/027afb5687a04f3cb770de8c58f62ade first=row-test-1733316396597-3-0 last=row-test-1733316396597-3-5 2024-12-04T12:48:09,559 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob in region [hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443] 2024-12-04T12:48:09,562 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/6759befaa3d44e67a2be163409a2ab50 first=row-test-1733316396597-3-0 last=row-test-1733316396597-3-5 2024-12-04T12:48:09,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'table1', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:48:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-12-04T12:48:09,569 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T12:48:09,569 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:09,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 62 2024-12-04T12:48:09,570 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T12:48:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T12:48:09,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741971_1147 (size=938) 2024-12-04T12:48:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T12:48:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T12:48:09,981 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a0802e74b13fc19a48cf81272df23d9f, NAME => 'table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f.', STARTKEY => '', ENDKEY => 'row0'}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:48:09,982 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => afd42af926399bd190cafc65ad0b947c, NAME => 'table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c.', STARTKEY => 'row0', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:48:09,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741972_1148 (size=45) 2024-12-04T12:48:09,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741973_1149 (size=45) 2024-12-04T12:48:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T12:48:10,395 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:48:10,395 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(898): Instantiated table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:48:10,395 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing a0802e74b13fc19a48cf81272df23d9f, disabling compactions & flushes 2024-12-04T12:48:10,395 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1722): Closing afd42af926399bd190cafc65ad0b947c, disabling compactions & flushes 2024-12-04T12:48:10,395 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. 2024-12-04T12:48:10,395 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1755): Closing region table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. 2024-12-04T12:48:10,395 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. 2024-12-04T12:48:10,396 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. 2024-12-04T12:48:10,396 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. after waiting 0 ms 2024-12-04T12:48:10,396 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. after waiting 0 ms 2024-12-04T12:48:10,396 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. 2024-12-04T12:48:10,396 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. 2024-12-04T12:48:10,396 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. 2024-12-04T12:48:10,396 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1973): Closed table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. 2024-12-04T12:48:10,396 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for a0802e74b13fc19a48cf81272df23d9f: Waiting for close lock at 1733316490395Disabling compacts and flushes for region at 1733316490395Disabling writes for close at 1733316490396 (+1 ms)Writing region close event to WAL at 1733316490396Closed at 1733316490396 2024-12-04T12:48:10,396 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1676): Region close journal for afd42af926399bd190cafc65ad0b947c: Waiting for close lock at 1733316490395Disabling compacts and flushes for region at 1733316490395Disabling writes for close at 1733316490396 (+1 ms)Writing region close event to WAL at 1733316490396Closed at 1733316490396 2024-12-04T12:48:10,400 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T12:48:10,401 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733316490400"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733316490400"}]},"ts":"1733316490400"} 2024-12-04T12:48:10,401 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733316490400"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733316490400"}]},"ts":"1733316490400"} 2024-12-04T12:48:10,406 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 2 regions to meta. 2024-12-04T12:48:10,407 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T12:48:10,407 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316490407"}]},"ts":"1733316490407"} 2024-12-04T12:48:10,409 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-12-04T12:48:10,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=a0802e74b13fc19a48cf81272df23d9f, ASSIGN}, {pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=afd42af926399bd190cafc65ad0b947c, ASSIGN}] 2024-12-04T12:48:10,410 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=afd42af926399bd190cafc65ad0b947c, ASSIGN 2024-12-04T12:48:10,411 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=a0802e74b13fc19a48cf81272df23d9f, ASSIGN 2024-12-04T12:48:10,411 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=a0802e74b13fc19a48cf81272df23d9f, ASSIGN; state=OFFLINE, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=false 2024-12-04T12:48:10,411 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=afd42af926399bd190cafc65ad0b947c, ASSIGN; state=OFFLINE, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=false 2024-12-04T12:48:10,562 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=afd42af926399bd190cafc65ad0b947c, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:10,562 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=a0802e74b13fc19a48cf81272df23d9f, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:10,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=a0802e74b13fc19a48cf81272df23d9f, ASSIGN because future has completed 2024-12-04T12:48:10,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0802e74b13fc19a48cf81272df23d9f, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:48:10,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=afd42af926399bd190cafc65ad0b947c, ASSIGN because future has completed 2024-12-04T12:48:10,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure afd42af926399bd190cafc65ad0b947c, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:48:10,588 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region c988c84788a01c971b5789026625491a, had cached 0 bytes from a total of 13547 2024-12-04T12:48:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T12:48:10,725 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. 2024-12-04T12:48:10,725 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => a0802e74b13fc19a48cf81272df23d9f, NAME => 'table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f.', STARTKEY => '', ENDKEY => 'row0'} 2024-12-04T12:48:10,726 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:48:10,726 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:10,726 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:48:10,727 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:10,727 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:10,728 INFO [StoreOpener-a0802e74b13fc19a48cf81272df23d9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:10,730 INFO [StoreOpener-a0802e74b13fc19a48cf81272df23d9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0802e74b13fc19a48cf81272df23d9f columnFamilyName f 2024-12-04T12:48:10,730 DEBUG [StoreOpener-a0802e74b13fc19a48cf81272df23d9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:10,731 INFO [StoreOpener-a0802e74b13fc19a48cf81272df23d9f-1 {}] regionserver.HStore(327): Store=a0802e74b13fc19a48cf81272df23d9f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:10,731 INFO [StoreOpener-a0802e74b13fc19a48cf81272df23d9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:10,732 INFO [StoreOpener-a0802e74b13fc19a48cf81272df23d9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0802e74b13fc19a48cf81272df23d9f columnFamilyName f3 2024-12-04T12:48:10,732 DEBUG [StoreOpener-a0802e74b13fc19a48cf81272df23d9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:10,733 INFO [StoreOpener-a0802e74b13fc19a48cf81272df23d9f-1 {}] regionserver.HStore(327): Store=a0802e74b13fc19a48cf81272df23d9f/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:10,733 INFO [StoreOpener-a0802e74b13fc19a48cf81272df23d9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:10,734 INFO [StoreOpener-a0802e74b13fc19a48cf81272df23d9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0802e74b13fc19a48cf81272df23d9f columnFamilyName mob 2024-12-04T12:48:10,735 DEBUG [StoreOpener-a0802e74b13fc19a48cf81272df23d9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:10,736 INFO [StoreOpener-a0802e74b13fc19a48cf81272df23d9f-1 {}] regionserver.HStore(327): Store=a0802e74b13fc19a48cf81272df23d9f/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:10,736 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:10,737 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:10,738 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:10,739 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:10,739 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:10,740 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T12:48:10,742 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:10,745 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T12:48:10,745 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened a0802e74b13fc19a48cf81272df23d9f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10071597600, jitterRate=-0.062009379267692566}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T12:48:10,746 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:10,746 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for a0802e74b13fc19a48cf81272df23d9f: Running coprocessor pre-open hook at 1733316490727Writing region info on filesystem at 1733316490727Initializing all the Stores at 1733316490728 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316490728Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316490728Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316490728Cleaning up temporary data from old regions at 1733316490739 (+11 ms)Running coprocessor post-open hooks at 1733316490746 (+7 ms)Region opened successfully at 1733316490746 2024-12-04T12:48:10,748 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f., pid=65, masterSystemTime=1733316490722 2024-12-04T12:48:10,750 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. 2024-12-04T12:48:10,751 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. 2024-12-04T12:48:10,751 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] handler.AssignRegionHandler(132): Open table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. 2024-12-04T12:48:10,751 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(7752): Opening region: {ENCODED => afd42af926399bd190cafc65ad0b947c, NAME => 'table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c.', STARTKEY => 'row0', ENDKEY => ''} 2024-12-04T12:48:10,751 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=a0802e74b13fc19a48cf81272df23d9f, regionState=OPEN, openSeqNum=2, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:10,751 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:48:10,751 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:10,751 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(898): Instantiated table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:48:10,751 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(7794): checking encryption for afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:10,751 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(7797): checking classloading for afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:10,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0802e74b13fc19a48cf81272df23d9f, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:48:10,753 INFO [StoreOpener-afd42af926399bd190cafc65ad0b947c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:10,754 INFO [StoreOpener-afd42af926399bd190cafc65ad0b947c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afd42af926399bd190cafc65ad0b947c columnFamilyName f 2024-12-04T12:48:10,754 DEBUG [StoreOpener-afd42af926399bd190cafc65ad0b947c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:10,755 INFO [StoreOpener-afd42af926399bd190cafc65ad0b947c-1 {}] regionserver.HStore(327): Store=afd42af926399bd190cafc65ad0b947c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:10,755 INFO [StoreOpener-afd42af926399bd190cafc65ad0b947c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:10,756 INFO [StoreOpener-afd42af926399bd190cafc65ad0b947c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afd42af926399bd190cafc65ad0b947c columnFamilyName f3 2024-12-04T12:48:10,756 DEBUG [StoreOpener-afd42af926399bd190cafc65ad0b947c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:10,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=63 2024-12-04T12:48:10,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure a0802e74b13fc19a48cf81272df23d9f, server=938b79ed23df,38879,1733316387992 in 185 msec 2024-12-04T12:48:10,757 INFO [StoreOpener-afd42af926399bd190cafc65ad0b947c-1 {}] regionserver.HStore(327): Store=afd42af926399bd190cafc65ad0b947c/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:10,757 INFO [StoreOpener-afd42af926399bd190cafc65ad0b947c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:10,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=a0802e74b13fc19a48cf81272df23d9f, ASSIGN in 348 msec 2024-12-04T12:48:10,758 INFO [StoreOpener-afd42af926399bd190cafc65ad0b947c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afd42af926399bd190cafc65ad0b947c columnFamilyName mob 2024-12-04T12:48:10,759 DEBUG [StoreOpener-afd42af926399bd190cafc65ad0b947c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:10,759 INFO [StoreOpener-afd42af926399bd190cafc65ad0b947c-1 {}] regionserver.HStore(327): Store=afd42af926399bd190cafc65ad0b947c/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:10,760 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1038): replaying wal for afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:10,760 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:10,761 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:10,761 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1048): stopping wal replay for afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:10,761 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1060): Cleaning up temporary data for afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:10,762 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T12:48:10,763 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1093): writing seq id for afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:10,765 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T12:48:10,765 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1114): Opened afd42af926399bd190cafc65ad0b947c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11830441120, jitterRate=0.10179568827152252}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T12:48:10,765 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1122): Running coprocessor post-open hooks for afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:10,765 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1006): Region open journal for afd42af926399bd190cafc65ad0b947c: Running coprocessor pre-open hook at 1733316490751Writing region info on filesystem at 1733316490751Initializing all the Stores at 1733316490752 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316490752Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316490753 (+1 ms)Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316490753Cleaning up temporary data from old regions at 1733316490761 (+8 ms)Running coprocessor post-open hooks at 1733316490765 (+4 ms)Region opened successfully at 1733316490765 2024-12-04T12:48:10,766 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c., pid=66, masterSystemTime=1733316490722 2024-12-04T12:48:10,768 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. 2024-12-04T12:48:10,768 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] handler.AssignRegionHandler(153): Opened table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. 2024-12-04T12:48:10,768 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=afd42af926399bd190cafc65ad0b947c, regionState=OPEN, openSeqNum=2, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:10,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure afd42af926399bd190cafc65ad0b947c, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:48:10,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=64 2024-12-04T12:48:10,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure afd42af926399bd190cafc65ad0b947c, server=938b79ed23df,38879,1733316387992 in 200 msec 2024-12-04T12:48:10,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=62 2024-12-04T12:48:10,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=62, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=afd42af926399bd190cafc65ad0b947c, ASSIGN in 363 msec 2024-12-04T12:48:10,775 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T12:48:10,775 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316490775"}]},"ts":"1733316490775"} 2024-12-04T12:48:10,776 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-12-04T12:48:10,777 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T12:48:10,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 1.2100 sec 2024-12-04T12:48:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T12:48:11,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-12-04T12:48:11,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-04T12:48:11,727 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443 from hbase tables test-1733316396597 to tables table1 2024-12-04T12:48:11,727 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore test-1733316396597 into table1 2024-12-04T12:48:11,730 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316491727 from hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443 2024-12-04T12:48:11,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5672418e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:11,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:11,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:11,733 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:11,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:11,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:11,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aa0f598, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:11,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:11,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:11,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:11,735 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:11,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4555e902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:11,736 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:11,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:11,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@60962f8f 2024-12-04T12:48:11,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:11,739 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:11,740 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-04T12:48:11,740 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table table1 2024-12-04T12:48:11,740 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-04T12:48:11,741 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:11,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:11,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:11,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:11,746 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: 2024-12-04T12:48:11,746 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: row0 2024-12-04T12:48:11,746 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-12-04T12:48:11,746 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/partitions_02a886f3-722a-4316-ad69-a250f7dd456f 2024-12-04T12:48:11,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741974_1150 (size=177) 2024-12-04T12:48:12,155 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,156 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,909 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-9906832250426938135.jar 2024-12-04T12:48:12,909 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:48:12,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T12:48:12,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T12:48:12,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T12:48:12,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T12:48:12,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T12:48:12,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T12:48:12,913 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T12:48:12,913 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T12:48:12,913 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T12:48:12,913 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T12:48:12,915 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,915 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,915 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,915 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,916 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,916 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:12,916 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:48:12,916 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: table1 2024-12-04T12:48:12,916 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:48:12,916 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:197) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:12,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:12,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:12,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:12,917 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-12-04T12:48:12,917 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:48:12,927 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:48:12,937 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-12-04T12:48:13,019 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local1952903065_0012_58cdc620-66db-4a2b-a393-d98314948e2b/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T12:48:13,019 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local1952903065_0012_58cdc620-66db-4a2b-a393-d98314948e2b/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T12:48:13,068 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40 2024-12-04T12:48:13,071 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-04T12:48:13,106 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/6759befaa3d44e67a2be163409a2ab50 2024-12-04T12:48:13,109 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-04T12:48:13,129 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f3/027afb5687a04f3cb770de8c58f62ade 2024-12-04T12:48:13,132 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-04T12:48:13,160 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4154ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:13,160 DEBUG [pool-267-thread-1 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:13,160 DEBUG [pool-267-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:13,161 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:13,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:13,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:13,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24c98e7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:13,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:13,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:13,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:13,163 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:13,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c8032e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:13,164 DEBUG [pool-267-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:13,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:13,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:13,165 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-12-04T12:48:13,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:13,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1733316396597-3-0', locateType=CURRENT is [region=table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:13,167 INFO [pool-267-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-267-thread-1. 2024-12-04T12:48:13,168 DEBUG [pool-267-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:13,168 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:13,168 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:13,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:13,168 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733316396597-3-0] 2024-12-04T12:48:13,168 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 938b79ed23df 2024-12-04T12:48:13,173 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5268baa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:13,173 DEBUG [pool-267-thread-1 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:13,173 DEBUG [pool-267-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:13,173 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:13,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:13,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:13,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b3472d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:13,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:13,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:13,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:13,175 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:13,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3e5b6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:13,176 DEBUG [pool-267-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:13,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:13,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:13,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:13,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1733316396597-3-0', locateType=CURRENT is [region=table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:13,180 INFO [pool-267-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-267-thread-1. 2024-12-04T12:48:13,180 DEBUG [pool-267-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:13,180 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:13,180 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:13,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:13,180 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733316396597-3-0] 2024-12-04T12:48:13,180 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 938b79ed23df 2024-12-04T12:48:13,184 DEBUG [pool-267-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316491727/_temporary/0/_temporary/attempt_local1952903065_0012_r_000000_0/f3/d6fad5b018574fbca6024f7cc7a11430 is 58, key is row-test-1733316396597-3-0/f3:q1/1733316406715/Put/seqid=0 2024-12-04T12:48:13,186 WARN [Thread-1635 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [938b79ed23df:38879] Specified favored nodes: [938b79ed23df:38879] 2024-12-04T12:48:13,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741975_1151 (size=5530) 2024-12-04T12:48:13,591 DEBUG [pool-267-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316491727/_temporary/0/_temporary/attempt_local1952903065_0012_r_000000_0/mob/8e5e26414449456eabaab4cb638f9677 is 59, key is row-test-1733316396597-3-0/mob:q1/1733316406729/Put/seqid=0 2024-12-04T12:48:13,593 WARN [Thread-1637 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [938b79ed23df:38879] Specified favored nodes: [938b79ed23df:38879] 2024-12-04T12:48:13,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741976_1152 (size=5539) 2024-12-04T12:48:14,019 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d821c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:14,019 DEBUG [pool-267-thread-1 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:14,019 DEBUG [pool-267-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:14,019 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:14,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:14,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:14,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4db9068f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:14,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:14,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:14,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:14,021 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:14,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c132196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:14,022 DEBUG [pool-267-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:14,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:14,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:14,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:14,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:14,026 INFO [pool-267-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-267-thread-1. 2024-12-04T12:48:14,026 DEBUG [pool-267-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:14,026 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:14,026 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:14,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:14,026 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row0] 2024-12-04T12:48:14,026 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 938b79ed23df 2024-12-04T12:48:14,033 DEBUG [pool-267-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316491727/_temporary/0/_temporary/attempt_local1952903065_0012_r_000001_0/f/17fee9f3ad484d64b6dacc3882170161 is 37, key is row10/f:q1/1733316399288/Put/seqid=0 2024-12-04T12:48:14,035 WARN [Thread-1652 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [938b79ed23df:38879] Specified favored nodes: [938b79ed23df:38879] 2024-12-04T12:48:14,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741977_1153 (size=8742) 2024-12-04T12:48:15,067 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316491727 2024-12-04T12:48:15,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd62737, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,067 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:15,068 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:15,068 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:15,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:15,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:15,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a60ecab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:15,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:15,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,070 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:15,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@325c5e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,071 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:15,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:15,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:15,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:15,076 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316491727/_SUCCESS 2024-12-04T12:48:15,084 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:15,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:15,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6bdae981 2024-12-04T12:48:15,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:15,086 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:15,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:15,098 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-04T12:48:15,108 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316491727/f/17fee9f3ad484d64b6dacc3882170161 first=Optional[row0] last=Optional[row98] 2024-12-04T12:48:15,111 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316491727/f3/d6fad5b018574fbca6024f7cc7a11430 first=Optional[row-test-1733316396597-3-0] last=Optional[row-test-1733316396597-3-5] 2024-12-04T12:48:15,114 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316491727/mob/8e5e26414449456eabaab4cb638f9677 first=Optional[row-test-1733316396597-3-0] last=Optional[row-test-1733316396597-3-5] 2024-12-04T12:48:15,132 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316491727/f/17fee9f3ad484d64b6dacc3882170161 for inclusion in afd42af926399bd190cafc65ad0b947c/f 2024-12-04T12:48:15,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316491727/f3/d6fad5b018574fbca6024f7cc7a11430 for inclusion in a0802e74b13fc19a48cf81272df23d9f/f3 2024-12-04T12:48:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HStore(626): HFile bounds: first=row0 last=row98 2024-12-04T12:48:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HStore(628): Region bounds: first=row0 last= 2024-12-04T12:48:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HRegion(2603): Flush status journal for afd42af926399bd190cafc65ad0b947c: 2024-12-04T12:48:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(626): HFile bounds: first=row-test-1733316396597-3-0 last=row-test-1733316396597-3-5 2024-12-04T12:48:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-12-04T12:48:15,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316491727/mob/8e5e26414449456eabaab4cb638f9677 for inclusion in a0802e74b13fc19a48cf81272df23d9f/mob 2024-12-04T12:48:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316491727/f/17fee9f3ad484d64b6dacc3882170161 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__ju5t9fo6fj8i5u9gra0j8ajknrod5mjo3mqc5tg7us4j4qqo2mpcenmeh5ceo5ch/f/17fee9f3ad484d64b6dacc3882170161 2024-12-04T12:48:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__ju5t9fo6fj8i5u9gra0j8ajknrod5mjo3mqc5tg7us4j4qqo2mpcenmeh5ceo5ch/f/17fee9f3ad484d64b6dacc3882170161 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c/f/c1b9d8c899474271af8af8760ea4a7e3_SeqId_4_ 2024-12-04T12:48:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(626): HFile bounds: first=row-test-1733316396597-3-0 last=row-test-1733316396597-3-5 2024-12-04T12:48:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-12-04T12:48:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HRegion(2603): Flush status journal for a0802e74b13fc19a48cf81272df23d9f: 2024-12-04T12:48:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14346b3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:15,144 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:15,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316491727/f3/d6fad5b018574fbca6024f7cc7a11430 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__ju5t9fo6fj8i5u9gra0j8ajknrod5mjo3mqc5tg7us4j4qqo2mpcenmeh5ceo5ch/f3/d6fad5b018574fbca6024f7cc7a11430 2024-12-04T12:48:15,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:15,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c4aaa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:15,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:15,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,145 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:15,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32a7ef43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__ju5t9fo6fj8i5u9gra0j8ajknrod5mjo3mqc5tg7us4j4qqo2mpcenmeh5ceo5ch/f3/d6fad5b018574fbca6024f7cc7a11430 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/f3/91a7c5efb6e14d8fa36afda5044e65da_SeqId_4_ 2024-12-04T12:48:15,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:15,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4b2272ae 2024-12-04T12:48:15,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316491727/mob/8e5e26414449456eabaab4cb638f9677 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__ju5t9fo6fj8i5u9gra0j8ajknrod5mjo3mqc5tg7us4j4qqo2mpcenmeh5ceo5ch/mob/8e5e26414449456eabaab4cb638f9677 2024-12-04T12:48:15,148 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__ju5t9fo6fj8i5u9gra0j8ajknrod5mjo3mqc5tg7us4j4qqo2mpcenmeh5ceo5ch/mob/8e5e26414449456eabaab4cb638f9677 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/mob/1a853a0a8b92408787b0f18b1f5f584c_SeqId_4_ 2024-12-04T12:48:15,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:15,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@374ba8c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:15,151 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:15,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:15,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:15,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:15,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@221d0218, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:15,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:15,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,153 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:15,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec75034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:15,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:15,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4a50bc19 2024-12-04T12:48:15,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:15,155 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:15,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:15,157 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:15,158 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:15,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:15,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:15,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:15,169 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:15,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:15,172 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879. 2024-12-04T12:48:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,172 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:15,173 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__ju5t9fo6fj8i5u9gra0j8ajknrod5mjo3mqc5tg7us4j4qqo2mpcenmeh5ceo5ch/f/17fee9f3ad484d64b6dacc3882170161 into afd42af926399bd190cafc65ad0b947c/f as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c/f/c1b9d8c899474271af8af8760ea4a7e3_SeqId_4_ - updating store file list. 2024-12-04T12:48:15,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:15,179 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c/f/c1b9d8c899474271af8af8760ea4a7e3_SeqId_4_ into afd42af926399bd190cafc65ad0b947c/f 2024-12-04T12:48:15,179 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__ju5t9fo6fj8i5u9gra0j8ajknrod5mjo3mqc5tg7us4j4qqo2mpcenmeh5ceo5ch/f/17fee9f3ad484d64b6dacc3882170161 into afd42af926399bd190cafc65ad0b947c/f (new location: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c/f/c1b9d8c899474271af8af8760ea4a7e3_SeqId_4_) 2024-12-04T12:48:15,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879. 2024-12-04T12:48:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__ju5t9fo6fj8i5u9gra0j8ajknrod5mjo3mqc5tg7us4j4qqo2mpcenmeh5ceo5ch/f/17fee9f3ad484d64b6dacc3882170161 2024-12-04T12:48:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f6bc867, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:15,181 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:15,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:15,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:15,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25699f73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:15,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:15,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,183 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:15,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e26b790, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:15,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:15,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1851ad0f 2024-12-04T12:48:15,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4854e4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:15,186 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:15,186 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:15,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:15,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:15,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c0313fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:15,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:15,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:15,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:15,188 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:15,188 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:15,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46911e06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:15,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:15,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5ea90d7d 2024-12-04T12:48:15,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:15,191 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-04T12:48:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:15,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:15,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:15,193 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-04T12:48:15,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:15,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:15,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:15,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:15,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879. 2024-12-04T12:48:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:15,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__ju5t9fo6fj8i5u9gra0j8ajknrod5mjo3mqc5tg7us4j4qqo2mpcenmeh5ceo5ch/f3/d6fad5b018574fbca6024f7cc7a11430 into a0802e74b13fc19a48cf81272df23d9f/f3 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/f3/91a7c5efb6e14d8fa36afda5044e65da_SeqId_4_ - updating store file list. 2024-12-04T12:48:15,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:15,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:15,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/f3/91a7c5efb6e14d8fa36afda5044e65da_SeqId_4_ into a0802e74b13fc19a48cf81272df23d9f/f3 2024-12-04T12:48:15,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__ju5t9fo6fj8i5u9gra0j8ajknrod5mjo3mqc5tg7us4j4qqo2mpcenmeh5ceo5ch/f3/d6fad5b018574fbca6024f7cc7a11430 into a0802e74b13fc19a48cf81272df23d9f/f3 (new location: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/f3/91a7c5efb6e14d8fa36afda5044e65da_SeqId_4_) 2024-12-04T12:48:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__ju5t9fo6fj8i5u9gra0j8ajknrod5mjo3mqc5tg7us4j4qqo2mpcenmeh5ceo5ch/f3/d6fad5b018574fbca6024f7cc7a11430 2024-12-04T12:48:15,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__ju5t9fo6fj8i5u9gra0j8ajknrod5mjo3mqc5tg7us4j4qqo2mpcenmeh5ceo5ch/mob/8e5e26414449456eabaab4cb638f9677 into a0802e74b13fc19a48cf81272df23d9f/mob as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/mob/1a853a0a8b92408787b0f18b1f5f584c_SeqId_4_ - updating store file list. 2024-12-04T12:48:15,213 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879. 2024-12-04T12:48:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:15,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/mob/1a853a0a8b92408787b0f18b1f5f584c_SeqId_4_ into a0802e74b13fc19a48cf81272df23d9f/mob 2024-12-04T12:48:15,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__ju5t9fo6fj8i5u9gra0j8ajknrod5mjo3mqc5tg7us4j4qqo2mpcenmeh5ceo5ch/mob/8e5e26414449456eabaab4cb638f9677 into a0802e74b13fc19a48cf81272df23d9f/mob (new location: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/mob/1a853a0a8b92408787b0f18b1f5f584c_SeqId_4_) 2024-12-04T12:48:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__ju5t9fo6fj8i5u9gra0j8ajknrod5mjo3mqc5tg7us4j4qqo2mpcenmeh5ceo5ch/mob/8e5e26414449456eabaab4cb638f9677 2024-12-04T12:48:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@723fb545, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:15,221 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:15,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:15,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:15,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42d44f1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:15,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:15,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,222 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:15,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@715c50a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:15,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:15,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@40e1d8dc 2024-12-04T12:48:15,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:15,224 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-04T12:48:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:15,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:15,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:15,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-04T12:48:15,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:15,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:15,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:15,240 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879. 2024-12-04T12:48:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:15,244 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:48:15,244 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:197) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:15,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:15,244 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:15,245 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-12-04T12:48:15,245 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-12-04T12:48:15,245 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1733316406744 hdfs://localhost:41555/backupUT/backup_1733316406744/default/test-1733316396597/ 2024-12-04T12:48:15,245 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:41555/backupUT/backup_1733316406744/.backup.manifest 2024-12-04T12:48:15,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f., hostname=938b79ed23df,38879,1733316387992, seqNum=2 , the old value is region=table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f., hostname=938b79ed23df,38879,1733316387992, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=938b79ed23df:38879 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-04T12:48:15,246 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-04T12:48:15,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f., hostname=938b79ed23df,38879,1733316387992, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-04T12:48:15,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f., hostname=938b79ed23df,38879,1733316387992, seqNum=2 from cache 2024-12-04T12:48:15,248 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733316406744/.backup.manifest 2024-12-04T12:48:15,248 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'ns2:test-17333163965971' to 'ns2:table2' from full backup image hdfs://localhost:41555/backupUT/backup_1733316406744/ns2/test-17333163965971 2024-12-04T12:48:15,256 INFO [Time-limited test {}] util.RestoreTool(491): Creating target table 'ns2:table2' 2024-12-04T12:48:15,256 DEBUG [Time-limited test {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:41555/backupUT/backup_1733316406744/ns2/test-17333163965971/archive/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a 2024-12-04T12:48:15,256 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:41555/backupUT/backup_1733316406744/ns2/test-17333163965971/archive/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f in region [hdfs://localhost:41555/backupUT/backup_1733316406744/ns2/test-17333163965971/archive/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a] 2024-12-04T12:48:15,260 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:41555/backupUT/backup_1733316406744/ns2/test-17333163965971/archive/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/194e38d3f9c54ed88dcc2848906ca7b4 first=row0 last=row98 2024-12-04T12:48:15,261 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:48:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:table2 2024-12-04T12:48:15,264 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T12:48:15,264 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 67 2024-12-04T12:48:15,265 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T12:48:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-04T12:48:15,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741978_1154 (size=351) 2024-12-04T12:48:15,354 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'table1', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=938b79ed23df:38879 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 34 more 2024-12-04T12:48:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-04T12:48:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-04T12:48:15,676 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => df2cfadd8eadf907300286d81ee0cbf8, NAME => 'ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:48:15,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741979_1155 (size=37) 2024-12-04T12:48:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-04T12:48:16,088 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:48:16,088 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1722): Closing df2cfadd8eadf907300286d81ee0cbf8, disabling compactions & flushes 2024-12-04T12:48:16,088 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. 2024-12-04T12:48:16,088 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. 2024-12-04T12:48:16,088 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. after waiting 0 ms 2024-12-04T12:48:16,088 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. 2024-12-04T12:48:16,088 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1973): Closed ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. 2024-12-04T12:48:16,088 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1676): Region close journal for df2cfadd8eadf907300286d81ee0cbf8: Waiting for close lock at 1733316496088Disabling compacts and flushes for region at 1733316496088Disabling writes for close at 1733316496088Writing region close event to WAL at 1733316496088Closed at 1733316496088 2024-12-04T12:48:16,090 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T12:48:16,091 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1733316496090"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733316496090"}]},"ts":"1733316496090"} 2024-12-04T12:48:16,093 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T12:48:16,095 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T12:48:16,095 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316496095"}]},"ts":"1733316496095"} 2024-12-04T12:48:16,097 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2024-12-04T12:48:16,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=df2cfadd8eadf907300286d81ee0cbf8, ASSIGN}] 2024-12-04T12:48:16,099 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=df2cfadd8eadf907300286d81ee0cbf8, ASSIGN 2024-12-04T12:48:16,100 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=df2cfadd8eadf907300286d81ee0cbf8, ASSIGN; state=OFFLINE, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=false 2024-12-04T12:48:16,251 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=df2cfadd8eadf907300286d81ee0cbf8, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:16,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=df2cfadd8eadf907300286d81ee0cbf8, ASSIGN because future has completed 2024-12-04T12:48:16,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure df2cfadd8eadf907300286d81ee0cbf8, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:48:16,258 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-12-04T12:48:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-04T12:48:16,412 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. 2024-12-04T12:48:16,412 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => df2cfadd8eadf907300286d81ee0cbf8, NAME => 'ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8.', STARTKEY => '', ENDKEY => ''} 2024-12-04T12:48:16,412 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:48:16,413 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 df2cfadd8eadf907300286d81ee0cbf8 2024-12-04T12:48:16,413 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:48:16,413 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for df2cfadd8eadf907300286d81ee0cbf8 2024-12-04T12:48:16,413 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for df2cfadd8eadf907300286d81ee0cbf8 2024-12-04T12:48:16,414 INFO [StoreOpener-df2cfadd8eadf907300286d81ee0cbf8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region df2cfadd8eadf907300286d81ee0cbf8 2024-12-04T12:48:16,416 INFO [StoreOpener-df2cfadd8eadf907300286d81ee0cbf8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df2cfadd8eadf907300286d81ee0cbf8 columnFamilyName f 2024-12-04T12:48:16,416 DEBUG [StoreOpener-df2cfadd8eadf907300286d81ee0cbf8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:16,417 INFO [StoreOpener-df2cfadd8eadf907300286d81ee0cbf8-1 {}] regionserver.HStore(327): Store=df2cfadd8eadf907300286d81ee0cbf8/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:16,417 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for df2cfadd8eadf907300286d81ee0cbf8 2024-12-04T12:48:16,418 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8 2024-12-04T12:48:16,418 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8 2024-12-04T12:48:16,418 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for df2cfadd8eadf907300286d81ee0cbf8 2024-12-04T12:48:16,418 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for df2cfadd8eadf907300286d81ee0cbf8 2024-12-04T12:48:16,420 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for df2cfadd8eadf907300286d81ee0cbf8 2024-12-04T12:48:16,422 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T12:48:16,423 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened df2cfadd8eadf907300286d81ee0cbf8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10698278080, jitterRate=-0.0036452114582061768}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T12:48:16,423 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df2cfadd8eadf907300286d81ee0cbf8 2024-12-04T12:48:16,424 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for df2cfadd8eadf907300286d81ee0cbf8: Running coprocessor pre-open hook at 1733316496413Writing region info on filesystem at 1733316496413Initializing all the Stores at 1733316496414 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316496414Cleaning up temporary data from old regions at 1733316496418 (+4 ms)Running coprocessor post-open hooks at 1733316496423 (+5 ms)Region opened successfully at 1733316496424 (+1 ms) 2024-12-04T12:48:16,425 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8., pid=69, masterSystemTime=1733316496408 2024-12-04T12:48:16,427 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. 2024-12-04T12:48:16,427 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. 2024-12-04T12:48:16,427 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=df2cfadd8eadf907300286d81ee0cbf8, regionState=OPEN, openSeqNum=2, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:16,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure df2cfadd8eadf907300286d81ee0cbf8, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:48:16,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-04T12:48:16,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure df2cfadd8eadf907300286d81ee0cbf8, server=938b79ed23df,38879,1733316387992 in 174 msec 2024-12-04T12:48:16,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-12-04T12:48:16,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=df2cfadd8eadf907300286d81ee0cbf8, ASSIGN in 333 msec 2024-12-04T12:48:16,433 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T12:48:16,434 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316496433"}]},"ts":"1733316496433"} 2024-12-04T12:48:16,435 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2024-12-04T12:48:16,436 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T12:48:16,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:table2 in 1.1750 sec 2024-12-04T12:48:16,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2024-12-04T12:48:16,830 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_table2 Metrics about Tables on a single HBase RegionServer 2024-12-04T12:48:16,832 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2024-12-04T12:48:16,832 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2024-12-04T12:48:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-04T12:48:17,414 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:table2 completed 2024-12-04T12:48:17,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-12-04T12:48:17,418 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:41555/backupUT/backup_1733316406744/ns2/test-17333163965971/archive/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a from hbase tables ns2:test-17333163965971 to tables ns2:table2 2024-12-04T12:48:17,418 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-17333163965971 into ns2:table2 2024-12-04T12:48:17,419 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-ns2-table2-1733316497418 from hdfs://localhost:41555/backupUT/backup_1733316406744/ns2/test-17333163965971/archive/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a 2024-12-04T12:48:17,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b76c849, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:17,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:17,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:17,421 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:17,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:17,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:17,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117fa024, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:17,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:17,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:17,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:17,423 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:17,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2920e961, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:17,423 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:17,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:17,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@480135 2024-12-04T12:48:17,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:17,425 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:17,426 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-04T12:48:17,426 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:table2 2024-12-04T12:48:17,426 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-12-04T12:48:17,426 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:17,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:17,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:17,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:17,429 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:table2: 2024-12-04T12:48:17,429 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 1 reduce partitions to match current region count for all tables 2024-12-04T12:48:17,430 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/partitions_f212d6f8-097d-43e9-a2a1-46736ec640e0 2024-12-04T12:48:17,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741980_1156 (size=153) 2024-12-04T12:48:17,840 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:17,841 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:17,842 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,071 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 9dab1d763bdadc4adf3b254f1a913949, had cached 0 bytes from a total of 41499 2024-12-04T12:48:18,597 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-8930795099708953545.jar 2024-12-04T12:48:18,598 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,598 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,598 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,598 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,599 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,599 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,599 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,599 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,599 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,600 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:48:18,600 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T12:48:18,600 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T12:48:18,600 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T12:48:18,600 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T12:48:18,600 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T12:48:18,601 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T12:48:18,601 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T12:48:18,601 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T12:48:18,601 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T12:48:18,601 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T12:48:18,602 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,602 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,602 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,602 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,602 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,602 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:18,602 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:48:18,603 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:table2 2024-12-04T12:48:18,603 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:48:18,603 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:197) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:18,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:18,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:18,603 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:18,603 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-12-04T12:48:18,603 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:48:18,613 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:48:18,625 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-12-04T12:48:18,704 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local1997454707_0013_0384441e-4516-4a7b-9576-a2c22b7d110b/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T12:48:18,704 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local1997454707_0013_0384441e-4516-4a7b-9576-a2c22b7d110b/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T12:48:18,760 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:41555/backupUT/backup_1733316406744/ns2/test-17333163965971/archive/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/194e38d3f9c54ed88dcc2848906ca7b4 2024-12-04T12:48:18,763 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-04T12:48:18,790 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fe899a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:18,791 DEBUG [pool-272-thread-1 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:18,791 DEBUG [pool-272-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:18,792 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:18,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:18,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:18,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62839289, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:18,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:18,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:18,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:18,793 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:18,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b78fc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:18,795 DEBUG [pool-272-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:18,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:18,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:18,797 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:18,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='row0', locateType=CURRENT is [region=ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:18,799 INFO [pool-272-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-272-thread-1. 2024-12-04T12:48:18,799 DEBUG [pool-272-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:18,799 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:18,800 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:18,800 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:18,800 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row0] 2024-12-04T12:48:18,800 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 938b79ed23df 2024-12-04T12:48:18,806 DEBUG [pool-272-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-ns2-table2-1733316497418/_temporary/0/_temporary/attempt_local1997454707_0013_r_000000_0/f/90c877e6c32b4309844522103a3d1dec is 37, key is row10/f:q1/1733316401612/Put/seqid=0 2024-12-04T12:48:18,808 WARN [Thread-1719 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [938b79ed23df:38879] Specified favored nodes: [938b79ed23df:38879] 2024-12-04T12:48:18,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741981_1157 (size=8742) 2024-12-04T12:48:19,167 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:table2' 2024-12-04T12:48:19,752 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-ns2-table2-1733316497418 2024-12-04T12:48:19,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d5ddf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:19,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:19,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:19,755 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:19,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:19,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:19,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f7e618c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:19,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:19,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:19,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:19,756 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:19,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3ebbca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:19,757 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:19,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:19,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:19,759 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:19,762 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-ns2-table2-1733316497418/_SUCCESS 2024-12-04T12:48:19,765 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:19,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:19,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4b74672d 2024-12-04T12:48:19,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:19,768 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:19,772 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:19,774 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-12-04T12:48:19,780 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-ns2-table2-1733316497418/f/90c877e6c32b4309844522103a3d1dec first=Optional[row0] last=Optional[row98] 2024-12-04T12:48:19,783 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-ns2-table2-1733316497418/f/90c877e6c32b4309844522103a3d1dec for inclusion in df2cfadd8eadf907300286d81ee0cbf8/f 2024-12-04T12:48:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(626): HFile bounds: first=row0 last=row98 2024-12-04T12:48:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T12:48:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HRegion(2603): Flush status journal for df2cfadd8eadf907300286d81ee0cbf8: 2024-12-04T12:48:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-ns2-table2-1733316497418/f/90c877e6c32b4309844522103a3d1dec to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__ns2_table2__hs84t96g117qtvbqlt4nlema8lg85949ln4vskt1p6vh6rpk9d034sk1occ0u91f/f/90c877e6c32b4309844522103a3d1dec 2024-12-04T12:48:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__ns2_table2__hs84t96g117qtvbqlt4nlema8lg85949ln4vskt1p6vh6rpk9d034sk1occ0u91f/f/90c877e6c32b4309844522103a3d1dec as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8/f/e15d35d1e7684dbaaddedc158d448c55_SeqId_4_ 2024-12-04T12:48:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df02e99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:19,792 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:19,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:19,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:19,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f87a40f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:19,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:19,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:19,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:19,793 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:19,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@370b1b50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:19,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:19,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3e0a36c7 2024-12-04T12:48:19,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:19,796 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:19,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:19,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:19,798 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:19,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:19,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:19,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:19,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879. 2024-12-04T12:48:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:19,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:19,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__ns2_table2__hs84t96g117qtvbqlt4nlema8lg85949ln4vskt1p6vh6rpk9d034sk1occ0u91f/f/90c877e6c32b4309844522103a3d1dec into df2cfadd8eadf907300286d81ee0cbf8/f as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8/f/e15d35d1e7684dbaaddedc158d448c55_SeqId_4_ - updating store file list. 2024-12-04T12:48:19,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8/f/e15d35d1e7684dbaaddedc158d448c55_SeqId_4_ into df2cfadd8eadf907300286d81ee0cbf8/f 2024-12-04T12:48:19,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__ns2_table2__hs84t96g117qtvbqlt4nlema8lg85949ln4vskt1p6vh6rpk9d034sk1occ0u91f/f/90c877e6c32b4309844522103a3d1dec into df2cfadd8eadf907300286d81ee0cbf8/f (new location: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8/f/e15d35d1e7684dbaaddedc158d448c55_SeqId_4_) 2024-12-04T12:48:19,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__ns2_table2__hs84t96g117qtvbqlt4nlema8lg85949ln4vskt1p6vh6rpk9d034sk1occ0u91f/f/90c877e6c32b4309844522103a3d1dec 2024-12-04T12:48:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3096f3f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:19,825 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:19,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:19,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:19,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68852ec5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:19,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:19,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:19,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:19,826 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:19,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fc8a299, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:19,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:19,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d2d4cca 2024-12-04T12:48:19,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:19,828 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-04T12:48:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:19,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:19,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:19,831 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-04T12:48:19,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:19,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:19,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:19,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879. 2024-12-04T12:48:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:19,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:19,845 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:48:19,845 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:197) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:19,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:19,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:19,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:19,845 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-12-04T12:48:19,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8., hostname=938b79ed23df,38879,1733316387992, seqNum=2 , the old value is region=ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8., hostname=938b79ed23df,38879,1733316387992, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=938b79ed23df:38879 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-04T12:48:19,845 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-12-04T12:48:19,845 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1733316406744 hdfs://localhost:41555/backupUT/backup_1733316406744/ns2/test-17333163965971/ 2024-12-04T12:48:19,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8., hostname=938b79ed23df,38879,1733316387992, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-04T12:48:19,845 DEBUG [Time-limited test {}] impl.RestoreTablesClient(236): restoreStage finished 2024-12-04T12:48:19,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8., hostname=938b79ed23df,38879,1733316387992, seqNum=2 from cache 2024-12-04T12:48:19,846 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-04T12:48:19,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:19,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:19,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:19,870 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:41555/backupUT/backup_1733316476524/.backup.manifest 2024-12-04T12:48:19,872 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733316476524/.backup.manifest 2024-12-04T12:48:19,873 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:41555/backupUT/backup_1733316442145/.backup.manifest 2024-12-04T12:48:19,874 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733316442145/.backup.manifest 2024-12-04T12:48:19,874 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'test-1733316396597' to 'table1' from full backup image hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597 2024-12-04T12:48:19,876 DEBUG [Time-limited test {}] util.RestoreTool(295): Retrieved descriptor: 'test-1733316396597', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} thru backup_1733316476524 2024-12-04T12:48:19,880 INFO [Time-limited test {}] util.RestoreTool(480): Truncating exising target table 'table1', preserving region splits 2024-12-04T12:48:19,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable table1 2024-12-04T12:48:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=table1 2024-12-04T12:48:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T12:48:19,892 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316499892"}]},"ts":"1733316499892"} 2024-12-04T12:48:19,893 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=DISABLING in hbase:meta 2024-12-04T12:48:19,894 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set table1 to state=DISABLING 2024-12-04T12:48:19,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=table1}] 2024-12-04T12:48:19,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=a0802e74b13fc19a48cf81272df23d9f, UNASSIGN}, {pid=73, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=afd42af926399bd190cafc65ad0b947c, UNASSIGN}] 2024-12-04T12:48:19,901 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=afd42af926399bd190cafc65ad0b947c, UNASSIGN 2024-12-04T12:48:19,902 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=a0802e74b13fc19a48cf81272df23d9f, UNASSIGN 2024-12-04T12:48:19,902 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=a0802e74b13fc19a48cf81272df23d9f, regionState=CLOSING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:19,902 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=afd42af926399bd190cafc65ad0b947c, regionState=CLOSING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:19,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=a0802e74b13fc19a48cf81272df23d9f, UNASSIGN because future has completed 2024-12-04T12:48:19,905 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T12:48:19,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure a0802e74b13fc19a48cf81272df23d9f, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:48:19,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=afd42af926399bd190cafc65ad0b947c, UNASSIGN because future has completed 2024-12-04T12:48:19,906 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T12:48:19,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=73, state=RUNNABLE, hasLock=false; CloseRegionProcedure afd42af926399bd190cafc65ad0b947c, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:48:19,955 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'ns2:table2', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=938b79ed23df:38879 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 34 more 2024-12-04T12:48:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T12:48:20,059 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] handler.UnassignRegionHandler(122): Close a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:20,059 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T12:48:20,059 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1722): Closing a0802e74b13fc19a48cf81272df23d9f, disabling compactions & flushes 2024-12-04T12:48:20,059 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1755): Closing region table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. 2024-12-04T12:48:20,059 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. 2024-12-04T12:48:20,060 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1843): Acquired close lock on table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. after waiting 0 ms 2024-12-04T12:48:20,060 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1853): Updates disabled for region table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. 2024-12-04T12:48:20,070 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-12-04T12:48:20,071 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:20,071 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1973): Closed table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. 2024-12-04T12:48:20,072 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1676): Region close journal for a0802e74b13fc19a48cf81272df23d9f: Waiting for close lock at 1733316500059Running coprocessor pre-close hooks at 1733316500059Disabling compacts and flushes for region at 1733316500059Disabling writes for close at 1733316500060 (+1 ms)Writing region close event to WAL at 1733316500062 (+2 ms)Running coprocessor post-close hooks at 1733316500071 (+9 ms)Closed at 1733316500071 2024-12-04T12:48:20,075 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] handler.UnassignRegionHandler(157): Closed a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:20,075 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(122): Close afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:20,075 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T12:48:20,075 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1722): Closing afd42af926399bd190cafc65ad0b947c, disabling compactions & flushes 2024-12-04T12:48:20,075 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1755): Closing region table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. 2024-12-04T12:48:20,075 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. 2024-12-04T12:48:20,075 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. after waiting 0 ms 2024-12-04T12:48:20,075 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. 2024-12-04T12:48:20,076 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=a0802e74b13fc19a48cf81272df23d9f, regionState=CLOSED 2024-12-04T12:48:20,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure a0802e74b13fc19a48cf81272df23d9f, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:48:20,080 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-12-04T12:48:20,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=72 2024-12-04T12:48:20,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=72, state=SUCCESS, hasLock=false; CloseRegionProcedure a0802e74b13fc19a48cf81272df23d9f, server=938b79ed23df,38879,1733316387992 in 173 msec 2024-12-04T12:48:20,081 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:20,081 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1973): Closed table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. 2024-12-04T12:48:20,081 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1676): Region close journal for afd42af926399bd190cafc65ad0b947c: Waiting for close lock at 1733316500075Running coprocessor pre-close hooks at 1733316500075Disabling compacts and flushes for region at 1733316500075Disabling writes for close at 1733316500075Writing region close event to WAL at 1733316500076 (+1 ms)Running coprocessor post-close hooks at 1733316500081 (+5 ms)Closed at 1733316500081 2024-12-04T12:48:20,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=a0802e74b13fc19a48cf81272df23d9f, UNASSIGN in 181 msec 2024-12-04T12:48:20,083 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(157): Closed afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:20,084 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=afd42af926399bd190cafc65ad0b947c, regionState=CLOSED 2024-12-04T12:48:20,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=73, state=RUNNABLE, hasLock=false; CloseRegionProcedure afd42af926399bd190cafc65ad0b947c, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:48:20,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=73 2024-12-04T12:48:20,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=73, state=SUCCESS, hasLock=false; CloseRegionProcedure afd42af926399bd190cafc65ad0b947c, server=938b79ed23df,38879,1733316387992 in 180 msec 2024-12-04T12:48:20,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=71 2024-12-04T12:48:20,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=71, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=afd42af926399bd190cafc65ad0b947c, UNASSIGN in 188 msec 2024-12-04T12:48:20,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-04T12:48:20,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=table1 in 196 msec 2024-12-04T12:48:20,094 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316500094"}]},"ts":"1733316500094"} 2024-12-04T12:48:20,096 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=DISABLED in hbase:meta 2024-12-04T12:48:20,096 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set table1 to state=DISABLED 2024-12-04T12:48:20,097 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; DisableTableProcedure table=table1 in 211 msec 2024-12-04T12:48:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T12:48:20,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:table1 completed 2024-12-04T12:48:20,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.HMaster$6(2602): Client=jenkins//172.17.0.3 truncate table1 2024-12-04T12:48:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION, hasLock=false; TruncateTableProcedure (table=table1 preserveSplits=true) 2024-12-04T12:48:20,226 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(87): waiting for 'table1' regions in transition 2024-12-04T12:48:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T12:48:20,232 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:20,232 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:20,235 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/f, FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/f3, FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/mob, FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/recovered.edits] 2024-12-04T12:48:20,235 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c/f, FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c/f3, FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c/mob, FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c/recovered.edits] 2024-12-04T12:48:20,240 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c/f/c1b9d8c899474271af8af8760ea4a7e3_SeqId_4_ to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/archive/data/default/table1/afd42af926399bd190cafc65ad0b947c/f/c1b9d8c899474271af8af8760ea4a7e3_SeqId_4_ 2024-12-04T12:48:20,240 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/f3/91a7c5efb6e14d8fa36afda5044e65da_SeqId_4_ to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/archive/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/f3/91a7c5efb6e14d8fa36afda5044e65da_SeqId_4_ 2024-12-04T12:48:20,243 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/mob/1a853a0a8b92408787b0f18b1f5f584c_SeqId_4_ to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/archive/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/mob/1a853a0a8b92408787b0f18b1f5f584c_SeqId_4_ 2024-12-04T12:48:20,244 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c/recovered.edits/6.seqid to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/archive/data/default/table1/afd42af926399bd190cafc65ad0b947c/recovered.edits/6.seqid 2024-12-04T12:48:20,245 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/afd42af926399bd190cafc65ad0b947c 2024-12-04T12:48:20,246 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/recovered.edits/6.seqid to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/archive/data/default/table1/a0802e74b13fc19a48cf81272df23d9f/recovered.edits/6.seqid 2024-12-04T12:48:20,246 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/a0802e74b13fc19a48cf81272df23d9f 2024-12-04T12:48:20,246 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived table1 regions 2024-12-04T12:48:20,247 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/mobdir/data/default/table1/b4aaa492ee713125ae5609f2682a2d2f 2024-12-04T12:48:20,247 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/mobdir/data/default/table1/b4aaa492ee713125ae5609f2682a2d2f/mob] 2024-12-04T12:48:20,249 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/mobdir/data/default/table1/b4aaa492ee713125ae5609f2682a2d2f 2024-12-04T12:48:20,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38879 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-04T12:48:20,260 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 2 rows of table1 from hbase:meta 2024-12-04T12:48:20,263 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'table1' descriptor. 2024-12-04T12:48:20,264 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'table1' from region states. 2024-12-04T12:48:20,264 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733316500264"}]},"ts":"9223372036854775807"} 2024-12-04T12:48:20,264 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733316500264"}]},"ts":"9223372036854775807"} 2024-12-04T12:48:20,266 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 2 regions from META 2024-12-04T12:48:20,266 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a0802e74b13fc19a48cf81272df23d9f, NAME => 'table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f.', STARTKEY => '', ENDKEY => 'row0'}, {ENCODED => afd42af926399bd190cafc65ad0b947c, NAME => 'table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c.', STARTKEY => 'row0', ENDKEY => ''}] 2024-12-04T12:48:20,266 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'table1' as deleted. 2024-12-04T12:48:20,267 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733316500266"}]},"ts":"9223372036854775807"} 2024-12-04T12:48:20,268 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table table1 state from META 2024-12-04T12:48:20,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741982_1158 (size=938) 2024-12-04T12:48:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T12:48:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T12:48:20,682 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 27f82bd5e2581f50581a7b2528369896, NAME => 'table1,,1733316500251.27f82bd5e2581f50581a7b2528369896.', STARTKEY => '', ENDKEY => 'row0'}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:48:20,683 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fd6a270c6969dabc7992cc41cebfae1b, NAME => 'table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b.', STARTKEY => 'row0', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:48:20,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741983_1159 (size=45) 2024-12-04T12:48:20,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741984_1160 (size=45) 2024-12-04T12:48:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T12:48:21,094 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(898): Instantiated table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:48:21,094 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1733316500251.27f82bd5e2581f50581a7b2528369896.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:48:21,094 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1722): Closing fd6a270c6969dabc7992cc41cebfae1b, disabling compactions & flushes 2024-12-04T12:48:21,094 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing 27f82bd5e2581f50581a7b2528369896, disabling compactions & flushes 2024-12-04T12:48:21,094 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1755): Closing region table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:21,094 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:21,094 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:21,094 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:21,094 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. after waiting 0 ms 2024-12-04T12:48:21,094 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. after waiting 0 ms 2024-12-04T12:48:21,094 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:21,094 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:21,094 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1973): Closed table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:21,094 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:21,094 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1676): Region close journal for fd6a270c6969dabc7992cc41cebfae1b: Waiting for close lock at 1733316501094Disabling compacts and flushes for region at 1733316501094Disabling writes for close at 1733316501094Writing region close event to WAL at 1733316501094Closed at 1733316501094 2024-12-04T12:48:21,094 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for 27f82bd5e2581f50581a7b2528369896: Waiting for close lock at 1733316501094Disabling compacts and flushes for region at 1733316501094Disabling writes for close at 1733316501094Writing region close event to WAL at 1733316501094Closed at 1733316501094 2024-12-04T12:48:21,097 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733316501096"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733316501096"}]},"ts":"1733316501096"} 2024-12-04T12:48:21,097 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1733316500251.27f82bd5e2581f50581a7b2528369896.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733316501096"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733316501096"}]},"ts":"1733316501096"} 2024-12-04T12:48:21,101 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 2 regions to meta. 2024-12-04T12:48:21,103 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316501103"}]},"ts":"1733316501103"} 2024-12-04T12:48:21,106 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-12-04T12:48:21,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=27f82bd5e2581f50581a7b2528369896, ASSIGN}, {pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=fd6a270c6969dabc7992cc41cebfae1b, ASSIGN}] 2024-12-04T12:48:21,107 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=27f82bd5e2581f50581a7b2528369896, ASSIGN 2024-12-04T12:48:21,107 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=fd6a270c6969dabc7992cc41cebfae1b, ASSIGN 2024-12-04T12:48:21,108 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=27f82bd5e2581f50581a7b2528369896, ASSIGN; state=OFFLINE, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=false 2024-12-04T12:48:21,108 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=fd6a270c6969dabc7992cc41cebfae1b, ASSIGN; state=OFFLINE, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=false 2024-12-04T12:48:21,260 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=77 updating hbase:meta row=27f82bd5e2581f50581a7b2528369896, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:21,260 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=fd6a270c6969dabc7992cc41cebfae1b, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:21,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=27f82bd5e2581f50581a7b2528369896, ASSIGN because future has completed 2024-12-04T12:48:21,264 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=77, state=RUNNABLE, hasLock=false; OpenRegionProcedure 27f82bd5e2581f50581a7b2528369896, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:48:21,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=fd6a270c6969dabc7992cc41cebfae1b, ASSIGN because future has completed 2024-12-04T12:48:21,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd6a270c6969dabc7992cc41cebfae1b, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:48:21,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T12:48:21,422 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:21,422 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 27f82bd5e2581f50581a7b2528369896, NAME => 'table1,,1733316500251.27f82bd5e2581f50581a7b2528369896.', STARTKEY => '', ENDKEY => 'row0'} 2024-12-04T12:48:21,423 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:48:21,423 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:21,423 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated table1,,1733316500251.27f82bd5e2581f50581a7b2528369896.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:48:21,423 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:21,423 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:21,425 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:21,428 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27f82bd5e2581f50581a7b2528369896 columnFamilyName f 2024-12-04T12:48:21,428 DEBUG [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:21,428 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] regionserver.HStore(327): Store=27f82bd5e2581f50581a7b2528369896/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:21,429 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:21,430 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27f82bd5e2581f50581a7b2528369896 columnFamilyName f3 2024-12-04T12:48:21,430 DEBUG [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:21,431 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] regionserver.HStore(327): Store=27f82bd5e2581f50581a7b2528369896/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:21,431 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:21,432 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27f82bd5e2581f50581a7b2528369896 columnFamilyName mob 2024-12-04T12:48:21,434 DEBUG [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:21,434 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] regionserver.HStore(327): Store=27f82bd5e2581f50581a7b2528369896/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:21,435 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:21,435 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:21,436 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:21,437 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:21,437 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:21,438 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T12:48:21,439 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:21,441 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T12:48:21,442 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 27f82bd5e2581f50581a7b2528369896; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9508550240, jitterRate=-0.11444725096225739}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T12:48:21,442 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:21,443 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 27f82bd5e2581f50581a7b2528369896: Running coprocessor pre-open hook at 1733316501424Writing region info on filesystem at 1733316501424Initializing all the Stores at 1733316501425 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316501425Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316501425Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316501425Cleaning up temporary data from old regions at 1733316501437 (+12 ms)Running coprocessor post-open hooks at 1733316501442 (+5 ms)Region opened successfully at 1733316501443 (+1 ms) 2024-12-04T12:48:21,443 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., pid=79, masterSystemTime=1733316501418 2024-12-04T12:48:21,445 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:21,445 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:21,445 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:21,445 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => fd6a270c6969dabc7992cc41cebfae1b, NAME => 'table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b.', STARTKEY => 'row0', ENDKEY => ''} 2024-12-04T12:48:21,445 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=77 updating hbase:meta row=27f82bd5e2581f50581a7b2528369896, regionState=OPEN, openSeqNum=2, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:21,446 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:48:21,446 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:21,446 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:48:21,446 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:21,446 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:21,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=77, state=RUNNABLE, hasLock=false; OpenRegionProcedure 27f82bd5e2581f50581a7b2528369896, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:48:21,447 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:21,449 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd6a270c6969dabc7992cc41cebfae1b columnFamilyName f 2024-12-04T12:48:21,449 DEBUG [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:21,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=77 2024-12-04T12:48:21,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=77, state=SUCCESS, hasLock=false; OpenRegionProcedure 27f82bd5e2581f50581a7b2528369896, server=938b79ed23df,38879,1733316387992 in 184 msec 2024-12-04T12:48:21,449 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] regionserver.HStore(327): Store=fd6a270c6969dabc7992cc41cebfae1b/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:21,449 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:21,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=27f82bd5e2581f50581a7b2528369896, ASSIGN in 343 msec 2024-12-04T12:48:21,450 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd6a270c6969dabc7992cc41cebfae1b columnFamilyName f3 2024-12-04T12:48:21,450 DEBUG [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:21,451 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] regionserver.HStore(327): Store=fd6a270c6969dabc7992cc41cebfae1b/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:21,451 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:21,452 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd6a270c6969dabc7992cc41cebfae1b columnFamilyName mob 2024-12-04T12:48:21,452 DEBUG [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:21,453 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] regionserver.HStore(327): Store=fd6a270c6969dabc7992cc41cebfae1b/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:21,453 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:21,454 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:21,454 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:21,455 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:21,455 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:21,456 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T12:48:21,457 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:21,459 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/fd6a270c6969dabc7992cc41cebfae1b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T12:48:21,460 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened fd6a270c6969dabc7992cc41cebfae1b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11173647840, jitterRate=0.040627047419548035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T12:48:21,460 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:21,460 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for fd6a270c6969dabc7992cc41cebfae1b: Running coprocessor pre-open hook at 1733316501446Writing region info on filesystem at 1733316501446Initializing all the Stores at 1733316501447 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316501447Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316501447Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316501447Cleaning up temporary data from old regions at 1733316501455 (+8 ms)Running coprocessor post-open hooks at 1733316501460 (+5 ms)Region opened successfully at 1733316501460 2024-12-04T12:48:21,461 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b., pid=80, masterSystemTime=1733316501418 2024-12-04T12:48:21,463 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:21,463 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:21,463 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=fd6a270c6969dabc7992cc41cebfae1b, regionState=OPEN, openSeqNum=2, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:21,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd6a270c6969dabc7992cc41cebfae1b, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:48:21,465 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36049 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=938b79ed23df,38879,1733316387992, table=table1, region=fd6a270c6969dabc7992cc41cebfae1b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-04T12:48:21,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=78 2024-12-04T12:48:21,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=78, state=SUCCESS, hasLock=false; OpenRegionProcedure fd6a270c6969dabc7992cc41cebfae1b, server=938b79ed23df,38879,1733316387992 in 200 msec 2024-12-04T12:48:21,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=76 2024-12-04T12:48:21,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=76, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=fd6a270c6969dabc7992cc41cebfae1b, ASSIGN in 361 msec 2024-12-04T12:48:21,469 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316501469"}]},"ts":"1733316501469"} 2024-12-04T12:48:21,470 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-12-04T12:48:21,471 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(145): truncate 'table1' completed 2024-12-04T12:48:21,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; TruncateTableProcedure (table=table1 preserveSplits=true) in 1.2510 sec 2024-12-04T12:48:21,919 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-12-04T12:48:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T12:48:22,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: TRUNCATE, Table Name: default:table1 completed 2024-12-04T12:48:22,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-04T12:48:22,386 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/archive/data/default/test-1733316396597/64682aa8ca2d45d6ee9ace61337cabc4,hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443 from hbase tables test-1733316396597 to tables table1 2024-12-04T12:48:22,386 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore test-1733316396597 into table1 2024-12-04T12:48:22,388 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316502386 from hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/archive/data/default/test-1733316396597/64682aa8ca2d45d6ee9ace61337cabc4,hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443 2024-12-04T12:48:22,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@206a5e7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:22,389 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:22,389 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:22,390 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:22,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:22,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:22,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59bf6aea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:22,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:22,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:22,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:22,391 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:22,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b90979, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:22,392 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:22,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:22,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@613da2e 2024-12-04T12:48:22,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:22,394 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:22,395 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-04T12:48:22,395 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table table1 2024-12-04T12:48:22,395 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-04T12:48:22,395 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:22,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:22,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:22,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:22,399 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: 2024-12-04T12:48:22,399 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: row0 2024-12-04T12:48:22,399 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-12-04T12:48:22,399 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/partitions_a2fe1e58-7cc3-4874-8ee5-d65a37550a2f 2024-12-04T12:48:22,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741985_1161 (size=177) 2024-12-04T12:48:22,809 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:22,810 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:22,811 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,563 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-13544505854116454026.jar 2024-12-04T12:48:23,563 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,564 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,564 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,564 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,564 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,564 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,564 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,565 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,565 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,565 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:48:23,565 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T12:48:23,565 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T12:48:23,565 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T12:48:23,565 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T12:48:23,566 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T12:48:23,566 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T12:48:23,566 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T12:48:23,566 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T12:48:23,566 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T12:48:23,566 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T12:48:23,567 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,567 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,567 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,567 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,567 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,567 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:23,567 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:48:23,568 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: table1 2024-12-04T12:48:23,568 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:48:23,568 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:218) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:23,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:23,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:23,568 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:23,568 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-12-04T12:48:23,568 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:48:23,577 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:48:23,586 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-12-04T12:48:23,667 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local879650503_0014_737e47a4-1d8a-4c18-8704-9dfc48987b63/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T12:48:23,667 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local879650503_0014_737e47a4-1d8a-4c18-8704-9dfc48987b63/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T12:48:23,717 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/5799223a4841435fb10d4958b6b73deb 2024-12-04T12:48:23,719 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-04T12:48:23,754 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/ed2773986acc4ae18188f34dec668da1 2024-12-04T12:48:23,757 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-04T12:48:23,778 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f/de63a3baba294af8906a08a560deeb40 2024-12-04T12:48:23,780 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-04T12:48:23,808 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/archive/data/default/test-1733316396597/64682aa8ca2d45d6ee9ace61337cabc4/mob/d41d8cd98f00b204e9800998ecf8427e20241204f4e86055bf4b405ab07a7b9c2fae1c0b_87e3e36a673c628af1440164c1fd3443 2024-12-04T12:48:23,811 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-04T12:48:23,831 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:41555/backupUT/backup_1733316442145/default/test-1733316396597/archive/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/mob/6759befaa3d44e67a2be163409a2ab50 2024-12-04T12:48:23,833 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-04T12:48:23,863 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7168c74a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:23,863 DEBUG [pool-277-thread-1 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:23,864 DEBUG [pool-277-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:23,865 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:23,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:23,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:23,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49d27e0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:23,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:23,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:23,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:23,866 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:23,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65788aa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:23,867 DEBUG [pool-277-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:23,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:23,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:23,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:23,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1733316396597-1-0', locateType=CURRENT is [region=table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:23,871 INFO [pool-277-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-277-thread-1. 2024-12-04T12:48:23,872 DEBUG [pool-277-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:23,872 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:23,872 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:23,872 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:23,872 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733316396597-1-0] 2024-12-04T12:48:23,872 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 938b79ed23df 2024-12-04T12:48:23,881 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e6928b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:23,881 DEBUG [pool-277-thread-1 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:23,881 DEBUG [pool-277-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:23,882 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:23,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:23,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:23,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f94d94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:23,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:23,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:23,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:23,883 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:23,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1588af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:23,884 DEBUG [pool-277-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:23,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:23,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:23,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:23,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1733316396597-3-0', locateType=CURRENT is [region=table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:23,888 INFO [pool-277-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-277-thread-1. 2024-12-04T12:48:23,888 DEBUG [pool-277-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:23,888 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:23,888 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:23,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:23,888 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733316396597-3-0] 2024-12-04T12:48:23,888 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 938b79ed23df 2024-12-04T12:48:23,894 DEBUG [pool-277-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316502386/_temporary/0/_temporary/attempt_local879650503_0014_r_000000_0/f/814c31ee19954765bd84ef2bfa2e8d36 is 59, key is row-test-1733316396597-1-10/f:q1/1733316426782/Put/seqid=0 2024-12-04T12:48:23,895 WARN [Thread-1806 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [938b79ed23df:38879] Specified favored nodes: [938b79ed23df:38879] 2024-12-04T12:48:23,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741986_1162 (size=11124) 2024-12-04T12:48:24,301 DEBUG [pool-277-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316502386/_temporary/0/_temporary/attempt_local879650503_0014_r_000000_0/mob/733eba919005446c89223e504d1f1534 is 192, key is row-test-1733316396597-3-100/mob:q1/1733316427129/Put/seqid=0 2024-12-04T12:48:24,305 WARN [Thread-1808 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [938b79ed23df:38879] Specified favored nodes: [938b79ed23df:38879] 2024-12-04T12:48:24,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741987_1163 (size=13403) 2024-12-04T12:48:24,337 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50a8e561, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:24,337 DEBUG [pool-277-thread-1 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:24,338 DEBUG [pool-277-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:24,338 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:24,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:24,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:24,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e9a2e12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:24,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:24,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:24,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:24,340 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:24,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132ef577, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:24,341 DEBUG [pool-277-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:24,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:24,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:24,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:24,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:24,345 INFO [pool-277-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-277-thread-1. 2024-12-04T12:48:24,345 DEBUG [pool-277-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:24,345 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:24,346 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:24,346 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:24,346 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row0] 2024-12-04T12:48:24,346 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 938b79ed23df 2024-12-04T12:48:24,350 DEBUG [pool-277-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316502386/_temporary/0/_temporary/attempt_local879650503_0014_r_000001_0/f/b8af5b3ef7524ac69a3a5cfa9f5bb1f5 is 37, key is row10/f:q1/1733316399288/Put/seqid=0 2024-12-04T12:48:24,352 WARN [Thread-1823 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [938b79ed23df:38879] Specified favored nodes: [938b79ed23df:38879] 2024-12-04T12:48:24,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741988_1164 (size=8741) 2024-12-04T12:48:25,583 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T12:48:25,715 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316502386 2024-12-04T12:48:25,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36f4a7f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,715 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:25,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:25,716 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:25,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:25,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:25,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@206dcd78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:25,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:25,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,718 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:25,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e66dd9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,719 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:25,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:25,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:25,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:25,724 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316502386/_SUCCESS 2024-12-04T12:48:25,731 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:25,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:25,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ed6cc67 2024-12-04T12:48:25,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:25,733 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:25,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:25,738 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-04T12:48:25,743 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316502386/f/814c31ee19954765bd84ef2bfa2e8d36 first=Optional[row-test-1733316396597-1-0] last=Optional[row-test-1733316396597-1-98] 2024-12-04T12:48:25,746 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316502386/f/b8af5b3ef7524ac69a3a5cfa9f5bb1f5 first=Optional[row0] last=Optional[row98] 2024-12-04T12:48:25,749 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316502386/mob/733eba919005446c89223e504d1f1534 first=Optional[row-test-1733316396597-3-0] last=Optional[row-test-1733316396597-3-99] 2024-12-04T12:48:25,753 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316502386/f/b8af5b3ef7524ac69a3a5cfa9f5bb1f5 for inclusion in fd6a270c6969dabc7992cc41cebfae1b/f 2024-12-04T12:48:25,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316502386/f/814c31ee19954765bd84ef2bfa2e8d36 for inclusion in 27f82bd5e2581f50581a7b2528369896/f 2024-12-04T12:48:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HStore(626): HFile bounds: first=row0 last=row98 2024-12-04T12:48:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HStore(628): Region bounds: first=row0 last= 2024-12-04T12:48:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HRegion(2603): Flush status journal for fd6a270c6969dabc7992cc41cebfae1b: 2024-12-04T12:48:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316502386/f/b8af5b3ef7524ac69a3a5cfa9f5bb1f5 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__2eouibq52rnicm094a67u6120an18hinkjbseicf2b56mla839nne58ht5p67g98/f/b8af5b3ef7524ac69a3a5cfa9f5bb1f5 2024-12-04T12:48:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(626): HFile bounds: first=row-test-1733316396597-1-0 last=row-test-1733316396597-1-98 2024-12-04T12:48:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-12-04T12:48:25,760 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316502386/mob/733eba919005446c89223e504d1f1534 for inclusion in 27f82bd5e2581f50581a7b2528369896/mob 2024-12-04T12:48:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__2eouibq52rnicm094a67u6120an18hinkjbseicf2b56mla839nne58ht5p67g98/f/b8af5b3ef7524ac69a3a5cfa9f5bb1f5 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/fd6a270c6969dabc7992cc41cebfae1b/f/69009820ba6f4b77a339dc9ee18f7698_SeqId_4_ 2024-12-04T12:48:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c79bce2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:25,763 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:25,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:25,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:25,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ead0ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:25,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:25,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(626): HFile bounds: first=row-test-1733316396597-3-0 last=row-test-1733316396597-3-99 2024-12-04T12:48:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-12-04T12:48:25,764 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HRegion(2603): Flush status journal for 27f82bd5e2581f50581a7b2528369896: 2024-12-04T12:48:25,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b3b606, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316502386/f/814c31ee19954765bd84ef2bfa2e8d36 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__2eouibq52rnicm094a67u6120an18hinkjbseicf2b56mla839nne58ht5p67g98/f/814c31ee19954765bd84ef2bfa2e8d36 2024-12-04T12:48:25,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:25,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c72db2b 2024-12-04T12:48:25,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:25,767 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__2eouibq52rnicm094a67u6120an18hinkjbseicf2b56mla839nne58ht5p67g98/f/814c31ee19954765bd84ef2bfa2e8d36 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/f/e1483fa7c2b442ec8e98a3cf240c5a67_SeqId_4_ 2024-12-04T12:48:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316502386/mob/733eba919005446c89223e504d1f1534 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__2eouibq52rnicm094a67u6120an18hinkjbseicf2b56mla839nne58ht5p67g98/mob/733eba919005446c89223e504d1f1534 2024-12-04T12:48:25,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:25,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:25,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__2eouibq52rnicm094a67u6120an18hinkjbseicf2b56mla839nne58ht5p67g98/mob/733eba919005446c89223e504d1f1534 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/mob/0a10613eba0e4833ad9e88a2fbd0fe58_SeqId_4_ 2024-12-04T12:48:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a9d023, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:25,773 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:25,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:25,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:25,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b6ee3ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:25,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:25,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,775 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:25,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bb81f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:25,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:25,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@143f6dd 2024-12-04T12:48:25,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:25,778 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:25,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:25,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:25,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:25,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:25,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:25,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:25,791 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879. 2024-12-04T12:48:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:25,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:25,793 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__2eouibq52rnicm094a67u6120an18hinkjbseicf2b56mla839nne58ht5p67g98/f/b8af5b3ef7524ac69a3a5cfa9f5bb1f5 into fd6a270c6969dabc7992cc41cebfae1b/f as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/fd6a270c6969dabc7992cc41cebfae1b/f/69009820ba6f4b77a339dc9ee18f7698_SeqId_4_ - updating store file list. 2024-12-04T12:48:25,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:25,799 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/fd6a270c6969dabc7992cc41cebfae1b/f/69009820ba6f4b77a339dc9ee18f7698_SeqId_4_ into fd6a270c6969dabc7992cc41cebfae1b/f 2024-12-04T12:48:25,799 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__2eouibq52rnicm094a67u6120an18hinkjbseicf2b56mla839nne58ht5p67g98/f/b8af5b3ef7524ac69a3a5cfa9f5bb1f5 into fd6a270c6969dabc7992cc41cebfae1b/f (new location: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/fd6a270c6969dabc7992cc41cebfae1b/f/69009820ba6f4b77a339dc9ee18f7698_SeqId_4_) 2024-12-04T12:48:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__2eouibq52rnicm094a67u6120an18hinkjbseicf2b56mla839nne58ht5p67g98/f/b8af5b3ef7524ac69a3a5cfa9f5bb1f5 2024-12-04T12:48:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5744692e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:25,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:25,803 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:25,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:25,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:25,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2213e3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:25,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:25,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,804 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:25,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879. 2024-12-04T12:48:25,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e622791, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:25,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4f57b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:25,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@524ad09 2024-12-04T12:48:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:25,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:25,807 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:25,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:25,807 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-04T12:48:25,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:25,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c9e3c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:25,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:25,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:25,809 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:25,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b1830d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:25,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:25,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:25,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:25,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4e424390 2024-12-04T12:48:25,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:25,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-04T12:48:25,811 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:25,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:25,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:25,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:25,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:25,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:25,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:25,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:25,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:25,830 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879. 2024-12-04T12:48:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:25,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:25,834 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879. 2024-12-04T12:48:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,834 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:25,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__2eouibq52rnicm094a67u6120an18hinkjbseicf2b56mla839nne58ht5p67g98/f/814c31ee19954765bd84ef2bfa2e8d36 into 27f82bd5e2581f50581a7b2528369896/f as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/f/e1483fa7c2b442ec8e98a3cf240c5a67_SeqId_4_ - updating store file list. 2024-12-04T12:48:25,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/f/e1483fa7c2b442ec8e98a3cf240c5a67_SeqId_4_ into 27f82bd5e2581f50581a7b2528369896/f 2024-12-04T12:48:25,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__2eouibq52rnicm094a67u6120an18hinkjbseicf2b56mla839nne58ht5p67g98/f/814c31ee19954765bd84ef2bfa2e8d36 into 27f82bd5e2581f50581a7b2528369896/f (new location: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/f/e1483fa7c2b442ec8e98a3cf240c5a67_SeqId_4_) 2024-12-04T12:48:25,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__2eouibq52rnicm094a67u6120an18hinkjbseicf2b56mla839nne58ht5p67g98/f/814c31ee19954765bd84ef2bfa2e8d36 2024-12-04T12:48:25,843 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__2eouibq52rnicm094a67u6120an18hinkjbseicf2b56mla839nne58ht5p67g98/mob/733eba919005446c89223e504d1f1534 into 27f82bd5e2581f50581a7b2528369896/mob as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/mob/0a10613eba0e4833ad9e88a2fbd0fe58_SeqId_4_ - updating store file list. 2024-12-04T12:48:25,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/mob/0a10613eba0e4833ad9e88a2fbd0fe58_SeqId_4_ into 27f82bd5e2581f50581a7b2528369896/mob 2024-12-04T12:48:25,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__2eouibq52rnicm094a67u6120an18hinkjbseicf2b56mla839nne58ht5p67g98/mob/733eba919005446c89223e504d1f1534 into 27f82bd5e2581f50581a7b2528369896/mob (new location: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/mob/0a10613eba0e4833ad9e88a2fbd0fe58_SeqId_4_) 2024-12-04T12:48:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__2eouibq52rnicm094a67u6120an18hinkjbseicf2b56mla839nne58ht5p67g98/mob/733eba919005446c89223e504d1f1534 2024-12-04T12:48:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c0fcc63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:25,852 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:25,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:25,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:25,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab39ee4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:25,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:25,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,854 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:25,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae8f059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:25,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:25,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4bf616d1 2024-12-04T12:48:25,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:25,857 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-04T12:48:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:25,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:25,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:25,860 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-04T12:48:25,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:25,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:25,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:25,876 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879. 2024-12-04T12:48:25,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:25,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:25,878 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:48:25,878 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:218) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:25,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:25,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:25,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., hostname=938b79ed23df,38879,1733316387992, seqNum=2 , the old value is region=table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., hostname=938b79ed23df,38879,1733316387992, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=938b79ed23df:38879 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-04T12:48:25,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., hostname=938b79ed23df,38879,1733316387992, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-04T12:48:25,879 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-12-04T12:48:25,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., hostname=938b79ed23df,38879,1733316387992, seqNum=2 from cache 2024-12-04T12:48:25,879 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-04T12:48:25,898 INFO [Time-limited test {}] impl.RestoreTablesClient(181): Restoring 'test-1733316396597' to 'table1' from log dirs: hdfs://localhost:41555/backupUT/backup_1733316476524/default/test-1733316396597/f2/4922ab9176074c7f84f86d1e2869075e 2024-12-04T12:48:25,902 DEBUG [Time-limited test {}] util.RestoreTool(177): Found descriptor 'test-1733316396597', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} through backup_1733316476524 2024-12-04T12:48:25,904 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table table1 from 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:48:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=table1 2024-12-04T12:48:25,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741989_1165 (size=938) 2024-12-04T12:48:25,984 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'table1', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=938b79ed23df:38879 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 34 more 2024-12-04T12:48:26,322 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/.tabledesc/.tableinfo.0000000001.938 2024-12-04T12:48:26,322 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/.tabledesc/.tableinfo.0000000002.938 2024-12-04T12:48:26,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=table1}] 2024-12-04T12:48:26,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=27f82bd5e2581f50581a7b2528369896, REOPEN/MOVE}, {pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=fd6a270c6969dabc7992cc41cebfae1b, REOPEN/MOVE}] 2024-12-04T12:48:26,334 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=27f82bd5e2581f50581a7b2528369896, REOPEN/MOVE 2024-12-04T12:48:26,334 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=fd6a270c6969dabc7992cc41cebfae1b, REOPEN/MOVE 2024-12-04T12:48:26,334 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=27f82bd5e2581f50581a7b2528369896, regionState=CLOSING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:26,334 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=fd6a270c6969dabc7992cc41cebfae1b, regionState=CLOSING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:26,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=27f82bd5e2581f50581a7b2528369896, REOPEN/MOVE because future has completed 2024-12-04T12:48:26,336 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T12:48:26,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=83, state=RUNNABLE, hasLock=false; CloseRegionProcedure 27f82bd5e2581f50581a7b2528369896, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:48:26,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=fd6a270c6969dabc7992cc41cebfae1b, REOPEN/MOVE because future has completed 2024-12-04T12:48:26,337 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T12:48:26,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure fd6a270c6969dabc7992cc41cebfae1b, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:48:26,491 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] handler.UnassignRegionHandler(122): Close fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:26,491 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T12:48:26,491 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1722): Closing fd6a270c6969dabc7992cc41cebfae1b, disabling compactions & flushes 2024-12-04T12:48:26,491 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1755): Closing region table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:26,491 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:26,492 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. after waiting 0 ms 2024-12-04T12:48:26,492 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:26,502 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/fd6a270c6969dabc7992cc41cebfae1b/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-12-04T12:48:26,504 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:26,504 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1973): Closed table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:26,504 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1676): Region close journal for fd6a270c6969dabc7992cc41cebfae1b: Waiting for close lock at 1733316506491Running coprocessor pre-close hooks at 1733316506491Disabling compacts and flushes for region at 1733316506491Disabling writes for close at 1733316506492 (+1 ms)Writing region close event to WAL at 1733316506494 (+2 ms)Running coprocessor post-close hooks at 1733316506503 (+9 ms)Closed at 1733316506504 (+1 ms) 2024-12-04T12:48:26,504 WARN [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegionServer(3299): Not adding moved region record: fd6a270c6969dabc7992cc41cebfae1b to self. 2024-12-04T12:48:26,507 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] handler.UnassignRegionHandler(157): Closed fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:26,507 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:26,507 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T12:48:26,507 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 27f82bd5e2581f50581a7b2528369896, disabling compactions & flushes 2024-12-04T12:48:26,508 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:26,508 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:26,508 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. after waiting 0 ms 2024-12-04T12:48:26,508 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:26,508 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=fd6a270c6969dabc7992cc41cebfae1b, regionState=CLOSED 2024-12-04T12:48:26,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure fd6a270c6969dabc7992cc41cebfae1b, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:48:26,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=84 2024-12-04T12:48:26,514 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-12-04T12:48:26,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure fd6a270c6969dabc7992cc41cebfae1b, server=938b79ed23df,38879,1733316387992 in 174 msec 2024-12-04T12:48:26,514 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:26,514 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=fd6a270c6969dabc7992cc41cebfae1b, REOPEN/MOVE; state=CLOSED, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=true 2024-12-04T12:48:26,515 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:26,515 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 27f82bd5e2581f50581a7b2528369896: Waiting for close lock at 1733316506507Running coprocessor pre-close hooks at 1733316506507Disabling compacts and flushes for region at 1733316506507Disabling writes for close at 1733316506508 (+1 ms)Writing region close event to WAL at 1733316506509 (+1 ms)Running coprocessor post-close hooks at 1733316506514 (+5 ms)Closed at 1733316506515 (+1 ms) 2024-12-04T12:48:26,515 WARN [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: 27f82bd5e2581f50581a7b2528369896 to self. 2024-12-04T12:48:26,517 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:26,517 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=27f82bd5e2581f50581a7b2528369896, regionState=CLOSED 2024-12-04T12:48:26,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=83, state=RUNNABLE, hasLock=false; CloseRegionProcedure 27f82bd5e2581f50581a7b2528369896, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:48:26,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=83 2024-12-04T12:48:26,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=83, state=SUCCESS, hasLock=false; CloseRegionProcedure 27f82bd5e2581f50581a7b2528369896, server=938b79ed23df,38879,1733316387992 in 184 msec 2024-12-04T12:48:26,523 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=27f82bd5e2581f50581a7b2528369896, REOPEN/MOVE; state=CLOSED, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=true 2024-12-04T12:48:26,666 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=fd6a270c6969dabc7992cc41cebfae1b, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:26,666 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=27f82bd5e2581f50581a7b2528369896, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:26,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=27f82bd5e2581f50581a7b2528369896, REOPEN/MOVE because future has completed 2024-12-04T12:48:26,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=83, state=RUNNABLE, hasLock=false; OpenRegionProcedure 27f82bd5e2581f50581a7b2528369896, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:48:26,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=fd6a270c6969dabc7992cc41cebfae1b, REOPEN/MOVE because future has completed 2024-12-04T12:48:26,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd6a270c6969dabc7992cc41cebfae1b, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:48:26,833 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:26,834 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 27f82bd5e2581f50581a7b2528369896, NAME => 'table1,,1733316500251.27f82bd5e2581f50581a7b2528369896.', STARTKEY => '', ENDKEY => 'row0'} 2024-12-04T12:48:26,835 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:48:26,835 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:26,835 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated table1,,1733316500251.27f82bd5e2581f50581a7b2528369896.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:48:26,835 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:26,835 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:26,838 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:26,839 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27f82bd5e2581f50581a7b2528369896 columnFamilyName f 2024-12-04T12:48:26,839 DEBUG [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:26,847 DEBUG [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/f/e1483fa7c2b442ec8e98a3cf240c5a67_SeqId_4_ 2024-12-04T12:48:26,847 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] regionserver.HStore(327): Store=27f82bd5e2581f50581a7b2528369896/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:26,847 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:26,849 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27f82bd5e2581f50581a7b2528369896 columnFamilyName f2 2024-12-04T12:48:26,849 DEBUG [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:26,850 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] regionserver.HStore(327): Store=27f82bd5e2581f50581a7b2528369896/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:26,850 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:26,851 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27f82bd5e2581f50581a7b2528369896 columnFamilyName mob 2024-12-04T12:48:26,852 DEBUG [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:26,880 DEBUG [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/mob/0a10613eba0e4833ad9e88a2fbd0fe58_SeqId_4_ 2024-12-04T12:48:26,880 INFO [StoreOpener-27f82bd5e2581f50581a7b2528369896-1 {}] regionserver.HStore(327): Store=27f82bd5e2581f50581a7b2528369896/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:26,881 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:26,882 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:26,884 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:26,885 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:26,885 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:26,886 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T12:48:26,888 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:26,890 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 27f82bd5e2581f50581a7b2528369896; next sequenceid=7; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11223940000, jitterRate=0.04531086981296539}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T12:48:26,890 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:26,891 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 27f82bd5e2581f50581a7b2528369896: Running coprocessor pre-open hook at 1733316506836Writing region info on filesystem at 1733316506836Initializing all the Stores at 1733316506837 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316506837Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316506837Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316506837Cleaning up temporary data from old regions at 1733316506885 (+48 ms)Running coprocessor post-open hooks at 1733316506890 (+5 ms)Region opened successfully at 1733316506890 2024-12-04T12:48:26,892 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., pid=87, masterSystemTime=1733316506824 2024-12-04T12:48:26,894 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:26,894 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:26,894 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:26,894 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => fd6a270c6969dabc7992cc41cebfae1b, NAME => 'table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b.', STARTKEY => 'row0', ENDKEY => ''} 2024-12-04T12:48:26,895 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=27f82bd5e2581f50581a7b2528369896, regionState=OPEN, openSeqNum=7, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:26,895 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:48:26,895 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:26,895 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:48:26,895 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:26,895 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:26,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=83, state=RUNNABLE, hasLock=false; OpenRegionProcedure 27f82bd5e2581f50581a7b2528369896, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:48:26,897 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:26,897 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd6a270c6969dabc7992cc41cebfae1b columnFamilyName f 2024-12-04T12:48:26,898 DEBUG [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:26,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=83 2024-12-04T12:48:26,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=83, state=SUCCESS, hasLock=false; OpenRegionProcedure 27f82bd5e2581f50581a7b2528369896, server=938b79ed23df,38879,1733316387992 in 227 msec 2024-12-04T12:48:26,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=27f82bd5e2581f50581a7b2528369896, REOPEN/MOVE in 566 msec 2024-12-04T12:48:26,903 DEBUG [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/fd6a270c6969dabc7992cc41cebfae1b/f/69009820ba6f4b77a339dc9ee18f7698_SeqId_4_ 2024-12-04T12:48:26,903 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] regionserver.HStore(327): Store=fd6a270c6969dabc7992cc41cebfae1b/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:26,903 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:26,904 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd6a270c6969dabc7992cc41cebfae1b columnFamilyName f2 2024-12-04T12:48:26,905 DEBUG [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:26,905 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] regionserver.HStore(327): Store=fd6a270c6969dabc7992cc41cebfae1b/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:26,905 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:26,906 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd6a270c6969dabc7992cc41cebfae1b columnFamilyName mob 2024-12-04T12:48:26,906 DEBUG [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:26,906 INFO [StoreOpener-fd6a270c6969dabc7992cc41cebfae1b-1 {}] regionserver.HStore(327): Store=fd6a270c6969dabc7992cc41cebfae1b/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:26,907 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:26,907 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:26,908 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:26,909 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:26,909 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:26,909 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T12:48:26,910 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:26,911 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened fd6a270c6969dabc7992cc41cebfae1b; next sequenceid=7; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9923592000, jitterRate=-0.07579347491264343}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T12:48:26,911 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:26,911 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for fd6a270c6969dabc7992cc41cebfae1b: Running coprocessor pre-open hook at 1733316506895Writing region info on filesystem at 1733316506895Initializing all the Stores at 1733316506896 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316506896Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316506896Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316506897 (+1 ms)Cleaning up temporary data from old regions at 1733316506909 (+12 ms)Running coprocessor post-open hooks at 1733316506911 (+2 ms)Region opened successfully at 1733316506911 2024-12-04T12:48:26,912 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b., pid=88, masterSystemTime=1733316506824 2024-12-04T12:48:26,913 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:26,913 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:26,914 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=fd6a270c6969dabc7992cc41cebfae1b, regionState=OPEN, openSeqNum=7, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:26,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd6a270c6969dabc7992cc41cebfae1b, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:48:26,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=84 2024-12-04T12:48:26,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure fd6a270c6969dabc7992cc41cebfae1b, server=938b79ed23df,38879,1733316387992 in 244 msec 2024-12-04T12:48:26,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=82 2024-12-04T12:48:26,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=82, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=fd6a270c6969dabc7992cc41cebfae1b, REOPEN/MOVE in 585 msec 2024-12-04T12:48:26,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-04T12:48:26,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=table1 in 590 msec 2024-12-04T12:48:26,922 DEBUG [PEWorker-3 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=table1 2024-12-04T12:48:26,922 DEBUG [PEWorker-3 {}] backup.HFileArchiver(265): No files to dispose of in table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., family=f3 2024-12-04T12:48:26,923 DEBUG [PEWorker-3 {}] backup.HFileArchiver(265): No files to dispose of in table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b., family=f3 2024-12-04T12:48:26,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; ModifyTableProcedure table=table1 in 1.0190 sec 2024-12-04T12:48:26,926 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-12-04T12:48:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-04T12:48:26,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:table1 completed 2024-12-04T12:48:26,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-04T12:48:26,931 INFO [Time-limited test {}] util.RestoreTool(200): Changed table1 to: 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:48:26,931 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore incremental backup from directory hdfs://localhost:41555/backupUT/backup_1733316476524/default/test-1733316396597/f2/4922ab9176074c7f84f86d1e2869075e from hbase tables test-1733316396597 to tables table1 2024-12-04T12:48:26,931 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore test-1733316396597 into table1 2024-12-04T12:48:26,933 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316506931 from hdfs://localhost:41555/backupUT/backup_1733316476524/default/test-1733316396597/f2/4922ab9176074c7f84f86d1e2869075e 2024-12-04T12:48:26,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5530210f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:26,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:26,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:26,934 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:26,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:26,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:26,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@517c69c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:26,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:26,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:26,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:26,936 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:26,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f2c036b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:26,937 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:26,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:26,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64b9d489 2024-12-04T12:48:26,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:26,939 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:26,940 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-04T12:48:26,940 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table table1 2024-12-04T12:48:26,940 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-04T12:48:26,940 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:26,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:26,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:26,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:26,945 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: 2024-12-04T12:48:26,945 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: row0 2024-12-04T12:48:26,945 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-12-04T12:48:26,945 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/partitions_b3717827-fe71-4ccf-8d2e-b29061d65acd 2024-12-04T12:48:26,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741990_1166 (size=177) 2024-12-04T12:48:27,356 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:27,357 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:27,357 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,112 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-15067986313513860557.jar 2024-12-04T12:48:28,112 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,113 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,113 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,113 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,113 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,113 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,113 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,114 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,114 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,114 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:48:28,114 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T12:48:28,114 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T12:48:28,114 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T12:48:28,115 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T12:48:28,115 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T12:48:28,115 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T12:48:28,115 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T12:48:28,115 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T12:48:28,115 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T12:48:28,116 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T12:48:28,116 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,116 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,116 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,116 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,116 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,117 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:28,117 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:48:28,117 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: table1 2024-12-04T12:48:28,117 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:48:28,117 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.incrementalRestoreTable(RestoreTool.java:205) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:185) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:218) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:28,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:28,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:28,117 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:28,117 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-12-04T12:48:28,118 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:48:28,128 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:48:28,139 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-12-04T12:48:28,212 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local1800595210_0015_b80b33e7-264c-468c-877b-36e390e92041/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T12:48:28,212 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local1800595210_0015_b80b33e7-264c-468c-877b-36e390e92041/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T12:48:28,263 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:41555/backupUT/backup_1733316476524/default/test-1733316396597/f2/4922ab9176074c7f84f86d1e2869075e 2024-12-04T12:48:28,265 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-04T12:48:28,294 DEBUG [pool-281-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ac78cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:28,294 DEBUG [pool-281-thread-1 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:28,294 DEBUG [pool-281-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:28,295 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:28,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:28,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:28,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aac1cd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:28,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:28,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:28,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:28,298 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:28,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bdc4984, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:28,299 DEBUG [pool-281-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:28,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:28,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:28,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:28,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1733316396597-2-0', locateType=CURRENT is [region=table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., hostname=938b79ed23df,38879,1733316387992, seqNum=7] 2024-12-04T12:48:28,304 INFO [pool-281-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-281-thread-1. 2024-12-04T12:48:28,304 DEBUG [pool-281-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:28,304 DEBUG [pool-281-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:28,305 DEBUG [pool-281-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:28,305 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:28,305 DEBUG [pool-281-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733316396597-2-0] 2024-12-04T12:48:28,305 DEBUG [pool-281-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 938b79ed23df 2024-12-04T12:48:28,309 DEBUG [pool-281-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316506931/_temporary/0/_temporary/attempt_local1800595210_0015_r_000000_0/f2/4e3988e961be41b8b377eddfe5ad5e4b is 58, key is row-test-1733316396597-2-0/f2:q1/1733316471514/Put/seqid=0 2024-12-04T12:48:28,310 WARN [Thread-1887 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [938b79ed23df:38879] Specified favored nodes: [938b79ed23df:38879] 2024-12-04T12:48:28,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741991_1167 (size=5609) 2024-12-04T12:48:29,249 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316506931 2024-12-04T12:48:29,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5409850f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:29,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:29,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:29,251 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:29,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:29,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:29,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e4f2bc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:29,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:29,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:29,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:29,253 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:29,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f4b2b77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:29,254 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:29,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:29,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:29,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:29,260 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316506931/_SUCCESS 2024-12-04T12:48:29,263 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:29,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:29,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2cfa9d64 2024-12-04T12:48:29,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:29,265 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:29,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., hostname=938b79ed23df,38879,1733316387992, seqNum=7] 2024-12-04T12:48:29,271 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-04T12:48:29,277 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316506931/f2/4e3988e961be41b8b377eddfe5ad5e4b first=Optional[row-test-1733316396597-2-0] last=Optional[row-test-1733316396597-2-6] 2024-12-04T12:48:29,282 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316506931/f2/4e3988e961be41b8b377eddfe5ad5e4b for inclusion in 27f82bd5e2581f50581a7b2528369896/f2 2024-12-04T12:48:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(626): HFile bounds: first=row-test-1733316396597-2-0 last=row-test-1733316396597-2-6 2024-12-04T12:48:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-12-04T12:48:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HRegion(2603): Flush status journal for 27f82bd5e2581f50581a7b2528369896: 2024-12-04T12:48:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-default-table1-1733316506931/f2/4e3988e961be41b8b377eddfe5ad5e4b to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__3b9255aqcbtoufa213oj8p58bht6c00v52fjn0fkrrdajl49rcbjvpu2b1keoijq/f2/4e3988e961be41b8b377eddfe5ad5e4b 2024-12-04T12:48:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__3b9255aqcbtoufa213oj8p58bht6c00v52fjn0fkrrdajl49rcbjvpu2b1keoijq/f2/4e3988e961be41b8b377eddfe5ad5e4b as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/f2/9e689a83452b4e87a869c1a77d574bfe_SeqId_9_ 2024-12-04T12:48:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@521ed00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:29,291 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:29,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:29,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:29,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e2a68f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:29,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:29,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:29,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:29,292 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:29,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7d42b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:29,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:29,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@288ceef2 2024-12-04T12:48:29,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:29,294 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:29,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:29,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:29,297 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:29,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:29,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:29,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:29,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879. 2024-12-04T12:48:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:29,309 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:29,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__3b9255aqcbtoufa213oj8p58bht6c00v52fjn0fkrrdajl49rcbjvpu2b1keoijq/f2/4e3988e961be41b8b377eddfe5ad5e4b into 27f82bd5e2581f50581a7b2528369896/f2 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/f2/9e689a83452b4e87a869c1a77d574bfe_SeqId_9_ - updating store file list. 2024-12-04T12:48:29,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/f2/9e689a83452b4e87a869c1a77d574bfe_SeqId_9_ into 27f82bd5e2581f50581a7b2528369896/f2 2024-12-04T12:48:29,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__3b9255aqcbtoufa213oj8p58bht6c00v52fjn0fkrrdajl49rcbjvpu2b1keoijq/f2/4e3988e961be41b8b377eddfe5ad5e4b into 27f82bd5e2581f50581a7b2528369896/f2 (new location: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/f2/9e689a83452b4e87a869c1a77d574bfe_SeqId_9_) 2024-12-04T12:48:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__table1__3b9255aqcbtoufa213oj8p58bht6c00v52fjn0fkrrdajl49rcbjvpu2b1keoijq/f2/4e3988e961be41b8b377eddfe5ad5e4b 2024-12-04T12:48:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@428eb81d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:29,317 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:29,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:29,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:29,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c0feb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:29,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:29,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:29,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:29,319 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:29,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c2997c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:29,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:29,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7a695456 2024-12-04T12:48:29,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:29,321 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-04T12:48:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:29,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:29,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:29,323 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-04T12:48:29,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:29,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:29,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:29,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879. 2024-12-04T12:48:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:29,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:29,338 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:48:29,338 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.incrementalRestoreTable(RestoreTool.java:205) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:185) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:218) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:29,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:29,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:29,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:29,339 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-12-04T12:48:29,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., hostname=938b79ed23df,38879,1733316387992, seqNum=7 , the old value is region=table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., hostname=938b79ed23df,38879,1733316387992, seqNum=7, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=938b79ed23df:38879 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-04T12:48:29,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., hostname=938b79ed23df,38879,1733316387992, seqNum=7 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-04T12:48:29,339 INFO [Time-limited test {}] impl.RestoreTablesClient(187): test-1733316396597 has been successfully restored to table1 2024-12-04T12:48:29,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., hostname=938b79ed23df,38879,1733316387992, seqNum=7 from cache 2024-12-04T12:48:29,339 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-12-04T12:48:29,339 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1733316476524 hdfs://localhost:41555/backupUT/backup_1733316476524/default/test-1733316396597/ 2024-12-04T12:48:29,339 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-04T12:48:29,340 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:41555/backupUT/backup_1733316442145/.backup.manifest 2024-12-04T12:48:29,342 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733316442145/.backup.manifest 2024-12-04T12:48:29,342 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'ns2:test-17333163965971' to 'ns2:table2' from full backup image hdfs://localhost:41555/backupUT/backup_1733316442145/ns2/test-17333163965971 2024-12-04T12:48:29,345 DEBUG [Time-limited test {}] util.RestoreTool(295): Retrieved descriptor: 'ns2:test-17333163965971', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} thru backup_1733316476524 2024-12-04T12:48:29,349 INFO [Time-limited test {}] util.RestoreTool(480): Truncating exising target table 'ns2:table2', preserving region splits 2024-12-04T12:48:29,349 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable ns2:table2 2024-12-04T12:48:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=ns2:table2 2024-12-04T12:48:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-04T12:48:29,353 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316509353"}]},"ts":"1733316509353"} 2024-12-04T12:48:29,355 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2024-12-04T12:48:29,355 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set ns2:table2 to state=DISABLING 2024-12-04T12:48:29,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=ns2:table2}] 2024-12-04T12:48:29,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=df2cfadd8eadf907300286d81ee0cbf8, UNASSIGN}] 2024-12-04T12:48:29,358 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=df2cfadd8eadf907300286d81ee0cbf8, UNASSIGN 2024-12-04T12:48:29,359 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=91 updating hbase:meta row=df2cfadd8eadf907300286d81ee0cbf8, regionState=CLOSING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:29,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=df2cfadd8eadf907300286d81ee0cbf8, UNASSIGN because future has completed 2024-12-04T12:48:29,361 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T12:48:29,361 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; CloseRegionProcedure df2cfadd8eadf907300286d81ee0cbf8, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:48:29,445 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'table1', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=938b79ed23df:38879 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 34 more 2024-12-04T12:48:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-04T12:48:29,515 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] handler.UnassignRegionHandler(122): Close df2cfadd8eadf907300286d81ee0cbf8 2024-12-04T12:48:29,515 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T12:48:29,516 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1722): Closing df2cfadd8eadf907300286d81ee0cbf8, disabling compactions & flushes 2024-12-04T12:48:29,516 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1755): Closing region ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. 2024-12-04T12:48:29,516 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. 2024-12-04T12:48:29,516 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. after waiting 0 ms 2024-12-04T12:48:29,516 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. 2024-12-04T12:48:29,525 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-12-04T12:48:29,526 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:29,527 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1973): Closed ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. 2024-12-04T12:48:29,527 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1676): Region close journal for df2cfadd8eadf907300286d81ee0cbf8: Waiting for close lock at 1733316509515Running coprocessor pre-close hooks at 1733316509516 (+1 ms)Disabling compacts and flushes for region at 1733316509516Disabling writes for close at 1733316509516Writing region close event to WAL at 1733316509518 (+2 ms)Running coprocessor post-close hooks at 1733316509526 (+8 ms)Closed at 1733316509526 2024-12-04T12:48:29,529 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] handler.UnassignRegionHandler(157): Closed df2cfadd8eadf907300286d81ee0cbf8 2024-12-04T12:48:29,530 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=91 updating hbase:meta row=df2cfadd8eadf907300286d81ee0cbf8, regionState=CLOSED 2024-12-04T12:48:29,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE, hasLock=false; CloseRegionProcedure df2cfadd8eadf907300286d81ee0cbf8, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:48:29,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-04T12:48:29,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; CloseRegionProcedure df2cfadd8eadf907300286d81ee0cbf8, server=938b79ed23df,38879,1733316387992 in 172 msec 2024-12-04T12:48:29,538 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-04T12:48:29,538 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=df2cfadd8eadf907300286d81ee0cbf8, UNASSIGN in 179 msec 2024-12-04T12:48:29,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-04T12:48:29,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=ns2:table2 in 183 msec 2024-12-04T12:48:29,545 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316509544"}]},"ts":"1733316509544"} 2024-12-04T12:48:29,547 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2024-12-04T12:48:29,547 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set ns2:table2 to state=DISABLED 2024-12-04T12:48:29,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; DisableTableProcedure table=ns2:table2 in 199 msec 2024-12-04T12:48:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-04T12:48:29,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: ns2:table2 completed 2024-12-04T12:48:29,676 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] master.HMaster$6(2602): Client=jenkins//172.17.0.3 truncate ns2:table2 2024-12-04T12:48:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION, hasLock=false; TruncateTableProcedure (table=ns2:table2 preserveSplits=true) 2024-12-04T12:48:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T12:48:29,681 DEBUG [PEWorker-5 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:table2' regions in transition 2024-12-04T12:48:29,686 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8 2024-12-04T12:48:29,690 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8/f, FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8/recovered.edits] 2024-12-04T12:48:29,698 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8/f/e15d35d1e7684dbaaddedc158d448c55_SeqId_4_ to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/archive/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8/f/e15d35d1e7684dbaaddedc158d448c55_SeqId_4_ 2024-12-04T12:48:29,704 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8/recovered.edits/6.seqid to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/archive/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8/recovered.edits/6.seqid 2024-12-04T12:48:29,705 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/df2cfadd8eadf907300286d81ee0cbf8 2024-12-04T12:48:29,705 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived ns2:table2 regions 2024-12-04T12:48:29,709 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2024-12-04T12:48:29,711 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'ns2:table2' descriptor. 2024-12-04T12:48:29,712 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'ns2:table2' from region states. 2024-12-04T12:48:29,712 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733316509712"}]},"ts":"9223372036854775807"} 2024-12-04T12:48:29,714 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T12:48:29,714 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => df2cfadd8eadf907300286d81ee0cbf8, NAME => 'ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T12:48:29,714 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'ns2:table2' as deleted. 2024-12-04T12:48:29,714 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733316509714"}]},"ts":"9223372036854775807"} 2024-12-04T12:48:29,716 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table ns2:table2 state from META 2024-12-04T12:48:29,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741992_1168 (size=351) 2024-12-04T12:48:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T12:48:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T12:48:30,128 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3ef1e15068edf0d0c9bdc810858b3ba2, NAME => 'ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a 2024-12-04T12:48:30,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741993_1169 (size=37) 2024-12-04T12:48:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T12:48:30,544 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:48:30,544 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1722): Closing 3ef1e15068edf0d0c9bdc810858b3ba2, disabling compactions & flushes 2024-12-04T12:48:30,544 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2. 2024-12-04T12:48:30,544 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2. 2024-12-04T12:48:30,544 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2. after waiting 0 ms 2024-12-04T12:48:30,544 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2. 2024-12-04T12:48:30,544 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1973): Closed ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2. 2024-12-04T12:48:30,545 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3ef1e15068edf0d0c9bdc810858b3ba2: Waiting for close lock at 1733316510544Disabling compacts and flushes for region at 1733316510544Disabling writes for close at 1733316510544Writing region close event to WAL at 1733316510544Closed at 1733316510544 2024-12-04T12:48:30,549 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1733316510548"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733316510548"}]},"ts":"1733316510548"} 2024-12-04T12:48:30,552 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T12:48:30,553 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316510553"}]},"ts":"1733316510553"} 2024-12-04T12:48:30,555 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2024-12-04T12:48:30,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=3ef1e15068edf0d0c9bdc810858b3ba2, ASSIGN}] 2024-12-04T12:48:30,557 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=3ef1e15068edf0d0c9bdc810858b3ba2, ASSIGN 2024-12-04T12:48:30,558 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=3ef1e15068edf0d0c9bdc810858b3ba2, ASSIGN; state=OFFLINE, location=938b79ed23df,38879,1733316387992; forceNewPlan=false, retain=false 2024-12-04T12:48:30,709 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=3ef1e15068edf0d0c9bdc810858b3ba2, regionState=OPENING, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:30,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=3ef1e15068edf0d0c9bdc810858b3ba2, ASSIGN because future has completed 2024-12-04T12:48:30,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ef1e15068edf0d0c9bdc810858b3ba2, server=938b79ed23df,38879,1733316387992}] 2024-12-04T12:48:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T12:48:30,875 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2. 2024-12-04T12:48:30,875 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 3ef1e15068edf0d0c9bdc810858b3ba2, NAME => 'ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2.', STARTKEY => '', ENDKEY => ''} 2024-12-04T12:48:30,876 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T12:48:30,876 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 3ef1e15068edf0d0c9bdc810858b3ba2 2024-12-04T12:48:30,876 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T12:48:30,876 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 3ef1e15068edf0d0c9bdc810858b3ba2 2024-12-04T12:48:30,877 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 3ef1e15068edf0d0c9bdc810858b3ba2 2024-12-04T12:48:30,879 INFO [StoreOpener-3ef1e15068edf0d0c9bdc810858b3ba2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 3ef1e15068edf0d0c9bdc810858b3ba2 2024-12-04T12:48:30,881 INFO [StoreOpener-3ef1e15068edf0d0c9bdc810858b3ba2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ef1e15068edf0d0c9bdc810858b3ba2 columnFamilyName f 2024-12-04T12:48:30,881 DEBUG [StoreOpener-3ef1e15068edf0d0c9bdc810858b3ba2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:30,882 INFO [StoreOpener-3ef1e15068edf0d0c9bdc810858b3ba2-1 {}] regionserver.HStore(327): Store=3ef1e15068edf0d0c9bdc810858b3ba2/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T12:48:30,882 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 3ef1e15068edf0d0c9bdc810858b3ba2 2024-12-04T12:48:30,883 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/3ef1e15068edf0d0c9bdc810858b3ba2 2024-12-04T12:48:30,883 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/3ef1e15068edf0d0c9bdc810858b3ba2 2024-12-04T12:48:30,884 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 3ef1e15068edf0d0c9bdc810858b3ba2 2024-12-04T12:48:30,884 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 3ef1e15068edf0d0c9bdc810858b3ba2 2024-12-04T12:48:30,886 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 3ef1e15068edf0d0c9bdc810858b3ba2 2024-12-04T12:48:30,889 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/3ef1e15068edf0d0c9bdc810858b3ba2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T12:48:30,890 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 3ef1e15068edf0d0c9bdc810858b3ba2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9804192160, jitterRate=-0.08691345155239105}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T12:48:30,890 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3ef1e15068edf0d0c9bdc810858b3ba2 2024-12-04T12:48:30,891 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 3ef1e15068edf0d0c9bdc810858b3ba2: Running coprocessor pre-open hook at 1733316510877Writing region info on filesystem at 1733316510877Initializing all the Stores at 1733316510878 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733316510878Cleaning up temporary data from old regions at 1733316510884 (+6 ms)Running coprocessor post-open hooks at 1733316510890 (+6 ms)Region opened successfully at 1733316510891 (+1 ms) 2024-12-04T12:48:30,892 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2., pid=95, masterSystemTime=1733316510867 2024-12-04T12:48:30,893 DEBUG [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2. 2024-12-04T12:48:30,893 INFO [RS_OPEN_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2. 2024-12-04T12:48:30,894 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=3ef1e15068edf0d0c9bdc810858b3ba2, regionState=OPEN, openSeqNum=2, regionLocation=938b79ed23df,38879,1733316387992 2024-12-04T12:48:30,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ef1e15068edf0d0c9bdc810858b3ba2, server=938b79ed23df,38879,1733316387992 because future has completed 2024-12-04T12:48:30,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-04T12:48:30,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 3ef1e15068edf0d0c9bdc810858b3ba2, server=938b79ed23df,38879,1733316387992 in 183 msec 2024-12-04T12:48:30,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-04T12:48:30,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=3ef1e15068edf0d0c9bdc810858b3ba2, ASSIGN in 342 msec 2024-12-04T12:48:30,900 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733316510899"}]},"ts":"1733316510899"} 2024-12-04T12:48:30,901 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2024-12-04T12:48:30,901 DEBUG [PEWorker-4 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:table2' completed 2024-12-04T12:48:30,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; TruncateTableProcedure (table=ns2:table2 preserveSplits=true) in 1.2240 sec 2024-12-04T12:48:31,177 DEBUG [regionserver/938b79ed23df:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-12-04T12:48:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T12:48:31,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: TRUNCATE, Table Name: ns2:table2 completed 2024-12-04T12:48:31,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-12-04T12:48:31,834 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:41555/backupUT/backup_1733316442145/ns2/test-17333163965971/archive/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a from hbase tables ns2:test-17333163965971 to tables ns2:table2 2024-12-04T12:48:31,834 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-17333163965971 into ns2:table2 2024-12-04T12:48:31,838 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-ns2-table2-1733316511835 from hdfs://localhost:41555/backupUT/backup_1733316442145/ns2/test-17333163965971/archive/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a 2024-12-04T12:48:31,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d4b5776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:31,839 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:31,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:31,840 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:31,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:31,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:31,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f4cea07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:31,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:31,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:31,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:31,841 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:31,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@791ed244, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:31,842 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:31,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:31,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@269c990a 2024-12-04T12:48:31,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:31,844 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:31,845 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-04T12:48:31,845 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:table2 2024-12-04T12:48:31,845 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-12-04T12:48:31,845 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:31,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:31,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:31,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:31,848 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:table2: 2024-12-04T12:48:31,848 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 1 reduce partitions to match current region count for all tables 2024-12-04T12:48:31,849 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/partitions_c57c3b38-18b3-434d-8d33-770d09f66219 2024-12-04T12:48:31,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741994_1170 (size=153) 2024-12-04T12:48:32,008 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-12-04T12:48:32,256 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:32,257 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:32,257 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,010 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop-3278367611648109819.jar 2024-12-04T12:48:33,010 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,011 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,011 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,011 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,011 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,012 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,012 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,012 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,012 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,012 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:48:33,012 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T12:48:33,013 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T12:48:33,013 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T12:48:33,013 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T12:48:33,013 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T12:48:33,013 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T12:48:33,013 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T12:48:33,014 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T12:48:33,014 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T12:48:33,014 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T12:48:33,014 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,014 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,015 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,015 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,015 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,015 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T12:48:33,015 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T12:48:33,015 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:table2 2024-12-04T12:48:33,015 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:48:33,015 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:218) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:33,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:33,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:33,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:33,016 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-12-04T12:48:33,016 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T12:48:33,025 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T12:48:33,036 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-12-04T12:48:33,126 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local816939887_0016_e14596f0-6482-405d-bbcc-af21a375e1d0/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T12:48:33,126 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/mapreduce.cluster.local.dir/job_local816939887_0016_e14596f0-6482-405d-bbcc-af21a375e1d0/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T12:48:33,171 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:41555/backupUT/backup_1733316442145/ns2/test-17333163965971/archive/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/194e38d3f9c54ed88dcc2848906ca7b4 2024-12-04T12:48:33,175 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-04T12:48:33,203 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:41555/backupUT/backup_1733316442145/ns2/test-17333163965971/archive/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/f/9fe180fa04404396b0429daa9427c60f 2024-12-04T12:48:33,206 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-04T12:48:33,231 DEBUG [pool-286-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b7ead3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:33,231 DEBUG [pool-286-thread-1 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:33,231 DEBUG [pool-286-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:33,232 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:33,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:33,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:33,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4737dfb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:33,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:33,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:33,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:33,233 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:33,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56f18d89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:33,234 DEBUG [pool-286-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:33,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:33,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:33,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:33,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='row-t20', locateType=CURRENT is [region=ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:33,238 INFO [pool-286-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-286-thread-1. 2024-12-04T12:48:33,238 DEBUG [pool-286-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:33,238 DEBUG [pool-286-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:33,238 DEBUG [pool-286-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:33,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:33,238 DEBUG [pool-286-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-12-04T12:48:33,238 DEBUG [pool-286-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 938b79ed23df 2024-12-04T12:48:33,242 DEBUG [pool-286-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-ns2-table2-1733316511835/_temporary/0/_temporary/attempt_local816939887_0016_r_000000_0/f/1226330861e44600b015a4019d339e62 is 38, key is row-t20/f:q1/1733316427166/Put/seqid=0 2024-12-04T12:48:33,244 WARN [Thread-1962 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [938b79ed23df:38879] Specified favored nodes: [938b79ed23df:38879] 2024-12-04T12:48:33,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741995_1171 (size=8934) 2024-12-04T12:48:34,169 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-ns2-table2-1733316511835 2024-12-04T12:48:34,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@165cb7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:34,170 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:34,170 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:34,171 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:34,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:34,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:34,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a6ea33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:34,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:34,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:34,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:34,172 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:34,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6636b77e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:34,173 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:34,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:34,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:34,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:34,177 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-ns2-table2-1733316511835/_SUCCESS 2024-12-04T12:48:34,180 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:34,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:34,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@48158a2e 2024-12-04T12:48:34,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:34,182 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:34,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:34,187 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-12-04T12:48:34,192 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-ns2-table2-1733316511835/f/1226330861e44600b015a4019d339e62 first=Optional[row-t20] last=Optional[row98] 2024-12-04T12:48:34,195 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-ns2-table2-1733316511835/f/1226330861e44600b015a4019d339e62 for inclusion in 3ef1e15068edf0d0c9bdc810858b3ba2/f 2024-12-04T12:48:34,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(626): HFile bounds: first=row-t20 last=row98 2024-12-04T12:48:34,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T12:48:34,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HRegion(2603): Flush status journal for 3ef1e15068edf0d0c9bdc810858b3ba2: 2024-12-04T12:48:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/hbase-staging/bulk_output-ns2-table2-1733316511835/f/1226330861e44600b015a4019d339e62 to hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__ns2_table2__bse1s0pafr51d92trre93hqvcqtrt0fj51tt7e30acj52pqfhshavl1as4nc1oes/f/1226330861e44600b015a4019d339e62 2024-12-04T12:48:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__ns2_table2__bse1s0pafr51d92trre93hqvcqtrt0fj51tt7e30acj52pqfhshavl1as4nc1oes/f/1226330861e44600b015a4019d339e62 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/3ef1e15068edf0d0c9bdc810858b3ba2/f/fd6bda4050e34491a392364cc432f5c6_SeqId_4_ 2024-12-04T12:48:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@454d8aa0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:34,203 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:34,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:34,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:34,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d16f70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:34,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:34,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:34,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:34,204 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:34,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ce5eca4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:34,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:34,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@283ef177 2024-12-04T12:48:34,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:34,207 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T12:48:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:34,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:34,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:34,209 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T12:48:34,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:34,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:34,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:34,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879. 2024-12-04T12:48:34,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:34,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:34,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:34,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:34,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__ns2_table2__bse1s0pafr51d92trre93hqvcqtrt0fj51tt7e30acj52pqfhshavl1as4nc1oes/f/1226330861e44600b015a4019d339e62 into 3ef1e15068edf0d0c9bdc810858b3ba2/f as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/3ef1e15068edf0d0c9bdc810858b3ba2/f/fd6bda4050e34491a392364cc432f5c6_SeqId_4_ - updating store file list. 2024-12-04T12:48:34,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/3ef1e15068edf0d0c9bdc810858b3ba2/f/fd6bda4050e34491a392364cc432f5c6_SeqId_4_ into 3ef1e15068edf0d0c9bdc810858b3ba2/f 2024-12-04T12:48:34,229 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__ns2_table2__bse1s0pafr51d92trre93hqvcqtrt0fj51tt7e30acj52pqfhshavl1as4nc1oes/f/1226330861e44600b015a4019d339e62 into 3ef1e15068edf0d0c9bdc810858b3ba2/f (new location: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/3ef1e15068edf0d0c9bdc810858b3ba2/f/fd6bda4050e34491a392364cc432f5c6_SeqId_4_) 2024-12-04T12:48:34,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/staging/jenkins__ns2_table2__bse1s0pafr51d92trre93hqvcqtrt0fj51tt7e30acj52pqfhshavl1as4nc1oes/f/1226330861e44600b015a4019d339e62 2024-12-04T12:48:34,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bbed16a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:34,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ClusterIdFetcher(90): Going to request 938b79ed23df,36049,-1 for getting cluster id 2024-12-04T12:48:34,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T12:48:34,231 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6f9b823b-072d-46db-86e7-8896bdc5736e' 2024-12-04T12:48:34,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T12:48:34,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6f9b823b-072d-46db-86e7-8896bdc5736e" 2024-12-04T12:48:34,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@793231d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:34,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [938b79ed23df,36049,-1] 2024-12-04T12:48:34,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T12:48:34,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:34,232 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-12-04T12:48:34,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ec1237f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T12:48:34,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T12:48:34,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 938b79ed23df,36049,1733316386472 2024-12-04T12:48:34,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@16e4f8ed 2024-12-04T12:48:34,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T12:48:34,235 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-04T12:48:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T12:48:34,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=938b79ed23df,38879,1733316387992, seqNum=-1] 2024-12-04T12:48:34,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T12:48:34,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-04T12:48:34,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T12:48:34,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T12:48:34,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:34,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879. 2024-12-04T12:48:34,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:34,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:34,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:34,250 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:34,251 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:48:34,251 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:218) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:34,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:34,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:34,251 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:34,251 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-12-04T12:48:34,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2., hostname=938b79ed23df,38879,1733316387992, seqNum=2 , the old value is region=ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2., hostname=938b79ed23df,38879,1733316387992, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=938b79ed23df:38879 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-04T12:48:34,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2., hostname=938b79ed23df,38879,1733316387992, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-04T12:48:34,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2., hostname=938b79ed23df,38879,1733316387992, seqNum=2 from cache 2024-12-04T12:48:34,252 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-04T12:48:34,266 INFO [Time-limited test {}] impl.RestoreTablesClient(175): No incremental changes since full backup for 'ns2:test-17333163965971', skipping incremental restore step. 2024-12-04T12:48:34,266 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-12-04T12:48:34,266 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1733316476524 hdfs://localhost:41555/backupUT/backup_1733316476524/ns2/test-17333163965971/ 2024-12-04T12:48:34,266 DEBUG [Time-limited test {}] impl.RestoreTablesClient(236): restoreStage finished 2024-12-04T12:48:34,267 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(222): After incremental restore: 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T12:48:34,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.CallRunner(138): callId: 742 service: ClientService methodName: Scan size: 160 connection: 172.17.0.3:44404 deadline: 1733316574267, exception=org.apache.hadoop.hbase.NotServingRegionException: table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. is not online on 938b79ed23df,38879,1733316387992 2024-12-04T12:48:34,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f., hostname=938b79ed23df,38879,1733316387992, seqNum=2 , the old value is region=table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f., hostname=938b79ed23df,38879,1733316387992, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. is not online on 938b79ed23df,38879,1733316387992 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:34,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f., hostname=938b79ed23df,38879,1733316387992, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f. is not online on 938b79ed23df,38879,1733316387992 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:34,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1733316489566.a0802e74b13fc19a48cf81272df23d9f., hostname=938b79ed23df,38879,1733316387992, seqNum=2 from cache 2024-12-04T12:48:34,354 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'ns2:table2', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=938b79ed23df:38879 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 34 more 2024-12-04T12:48:34,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1733316500251.27f82bd5e2581f50581a7b2528369896., hostname=938b79ed23df,38879,1733316387992, seqNum=7] 2024-12-04T12:48:34,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.CallRunner(138): callId: 746 service: ClientService methodName: Scan size: 170 connection: 172.17.0.3:44404 deadline: 1733316574378, exception=org.apache.hadoop.hbase.NotServingRegionException: table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. is not online on 938b79ed23df,38879,1733316387992 2024-12-04T12:48:34,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c., hostname=938b79ed23df,38879,1733316387992, seqNum=2 , the old value is region=table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c., hostname=938b79ed23df,38879,1733316387992, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. is not online on 938b79ed23df,38879,1733316387992 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:34,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c., hostname=938b79ed23df,38879,1733316387992, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c. is not online on 938b79ed23df,38879,1733316387992 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:34,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,row0,1733316489566.afd42af926399bd190cafc65ad0b947c., hostname=938b79ed23df,38879,1733316387992, seqNum=2 from cache 2024-12-04T12:48:34,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b., hostname=938b79ed23df,38879,1733316387992, seqNum=7] 2024-12-04T12:48:34,491 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(224): f1 has 198 rows 2024-12-04T12:48:34,497 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(228): f2 has 7 rows 2024-12-04T12:48:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:34,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-12-04T12:48:34,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:34,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-12-04T12:48:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:34,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-12-04T12:48:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:34,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-12-04T12:48:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:34,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-12-04T12:48:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T12:48:34,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-12-04T12:48:34,522 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(232): mob has 111 rows 2024-12-04T12:48:34,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38879 {}] ipc.CallRunner(138): callId: 754 service: ClientService methodName: Scan size: 159 connection: 172.17.0.3:44404 deadline: 1733316574522, exception=org.apache.hadoop.hbase.NotServingRegionException: ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. is not online on 938b79ed23df,38879,1733316387992 2024-12-04T12:48:34,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8., hostname=938b79ed23df,38879,1733316387992, seqNum=2 , the old value is region=ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8., hostname=938b79ed23df,38879,1733316387992, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. is not online on 938b79ed23df,38879,1733316387992 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:34,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8., hostname=938b79ed23df,38879,1733316387992, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8. is not online on 938b79ed23df,38879,1733316387992 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T12:48:34,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8., hostname=938b79ed23df,38879,1733316387992, seqNum=2 from cache 2024-12-04T12:48:34,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2., hostname=938b79ed23df,38879,1733316387992, seqNum=2] 2024-12-04T12:48:34,640 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:48:34,640 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:240) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:34,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:34,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:34,640 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:34,659 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=634 (was 611) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-788404336_20 at /127.0.0.1:54346 [Receiving block BP-117991357-172.17.0.3-1733316383096:blk_1073741956_1132] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/938b79ed23df:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(938b79ed23df,38879,1733316387992)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-117991357-172.17.0.3-1733316383096:blk_1073741955_1131, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: (938b79ed23df,36049,1733316386472)-proc-coordinator-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: member: '938b79ed23df,38879,1733316387992' subprocedure-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/cluster_4dfbfc9e-7777-d399-4020-07547724c305/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-788404336_20 at /127.0.0.1:54340 [Receiving block BP-117991357-172.17.0.3-1733316383096:blk_1073741955_1131] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/cluster_4dfbfc9e-7777-d399-4020-07547724c305/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-117991357-172.17.0.3-1733316383096:blk_1073741956_1132, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(938b79ed23df,38879,1733316387992)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-117991357-172.17.0.3-1733316383096:blk_1073741954_1130, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-358026100_20 at /127.0.0.1:51518 [Waiting for operation #115] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/938b79ed23df:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'JobTracker' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-788404336_20 at /127.0.0.1:54326 [Receiving block BP-117991357-172.17.0.3-1733316383096:blk_1073741954_1130] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(938b79ed23df,38879,1733316387992)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(938b79ed23df,38879,1733316387992)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=643 (was 633) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=78 (was 122), ProcessCount=14 (was 14), AvailableMemoryMB=6707 (was 7253) 2024-12-04T12:48:34,659 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=634 is superior to 500 2024-12-04T12:48:34,667 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T12:48:34,667 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:48:34,667 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:34,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:34,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:34,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:34,668 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T12:48:34,668 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1415824117, stopped=false 2024-12-04T12:48:34,669 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=938b79ed23df,36049,1733316386472 2024-12-04T12:48:34,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-12-04T12:48:34,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-12-04T12:48:34,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:48:34,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:48:34,712 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T12:48:34,713 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T12:48:34,714 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:34,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:34,714 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-04T12:48:34,715 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '938b79ed23df,38879,1733316387992' ***** 2024-12-04T12:48:34,715 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-04T12:48:34,715 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T12:48:34,715 INFO [RS:0;938b79ed23df:38879 {}] hbase.HBaseServerBase(468): Stop info server 2024-12-04T12:48:34,722 INFO [RS:0;938b79ed23df:38879 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@49ec94de{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-12-04T12:48:34,722 INFO [regionserver/938b79ed23df:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T12:48:34,722 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T12:48:34,724 INFO [RS:0;938b79ed23df:38879 {}] server.AbstractConnector(383): Stopped ServerConnector@13520b09{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-12-04T12:48:34,724 INFO [RS:0;938b79ed23df:38879 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T12:48:34,724 INFO [RS:0;938b79ed23df:38879 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@54a4c9e0{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-12-04T12:48:34,724 INFO [RS:0;938b79ed23df:38879 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@749d911f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,STOPPED} 2024-12-04T12:48:34,726 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T12:48:34,726 INFO [RS:0;938b79ed23df:38879 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T12:48:34,726 INFO [RS:0;938b79ed23df:38879 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2024-12-04T12:48:34,726 INFO [RS:0;938b79ed23df:38879 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T12:48:34,726 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(3091): Received CLOSE for d7d9cf8458b0e699dff32ea7d56f7d9d 2024-12-04T12:48:34,727 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(3091): Received CLOSE for fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:34,727 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(3091): Received CLOSE for 27f82bd5e2581f50581a7b2528369896 2024-12-04T12:48:34,727 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(3091): Received CLOSE for 87e3e36a673c628af1440164c1fd3443 2024-12-04T12:48:34,727 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(3091): Received CLOSE for 3ef1e15068edf0d0c9bdc810858b3ba2 2024-12-04T12:48:34,727 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(3091): Received CLOSE for c988c84788a01c971b5789026625491a 2024-12-04T12:48:34,727 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(3091): Received CLOSE for deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:48:34,727 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(3091): Received CLOSE for 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:48:34,727 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(3091): Received CLOSE for 58f768c3c54039235f5a6360aacc8ebf 2024-12-04T12:48:34,727 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(959): stopping server 938b79ed23df,38879,1733316387992 2024-12-04T12:48:34,727 INFO [RS:0;938b79ed23df:38879 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T12:48:34,727 INFO [RS:0;938b79ed23df:38879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;938b79ed23df:38879. 2024-12-04T12:48:34,727 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d7d9cf8458b0e699dff32ea7d56f7d9d, disabling compactions & flushes 2024-12-04T12:48:34,727 DEBUG [RS:0;938b79ed23df:38879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:34,727 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. 2024-12-04T12:48:34,727 DEBUG [RS:0;938b79ed23df:38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:34,727 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. 2024-12-04T12:48:34,727 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. after waiting 0 ms 2024-12-04T12:48:34,727 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. 2024-12-04T12:48:34,727 INFO [RS:0;938b79ed23df:38879 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T12:48:34,727 INFO [RS:0;938b79ed23df:38879 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T12:48:34,727 INFO [RS:0;938b79ed23df:38879 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T12:48:34,727 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T12:48:34,728 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-12-04T12:48:34,728 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T12:48:34,728 INFO [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T12:48:34,728 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T12:48:34,728 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T12:48:34,728 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T12:48:34,728 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 27f82bd5e2581f50581a7b2528369896, 3ef1e15068edf0d0c9bdc810858b3ba2, 58f768c3c54039235f5a6360aacc8ebf, 87e3e36a673c628af1440164c1fd3443, 9dab1d763bdadc4adf3b254f1a913949, c988c84788a01c971b5789026625491a, d7d9cf8458b0e699dff32ea7d56f7d9d, deae8c07fc7147a6340e830c87bd97da, fd6a270c6969dabc7992cc41cebfae1b 2024-12-04T12:48:34,728 INFO [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=23.97 KB heapSize=42.40 KB 2024-12-04T12:48:34,731 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns3/test-17333163965972/d7d9cf8458b0e699dff32ea7d56f7d9d/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-04T12:48:34,731 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:34,731 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. 2024-12-04T12:48:34,731 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d7d9cf8458b0e699dff32ea7d56f7d9d: Waiting for close lock at 1733316514727Running coprocessor pre-close hooks at 1733316514727Disabling compacts and flushes for region at 1733316514727Disabling writes for close at 1733316514727Writing region close event to WAL at 1733316514728 (+1 ms)Running coprocessor post-close hooks at 1733316514731 (+3 ms)Closed at 1733316514731 2024-12-04T12:48:34,732 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17333163965972,,1733316401764.d7d9cf8458b0e699dff32ea7d56f7d9d. 2024-12-04T12:48:34,732 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fd6a270c6969dabc7992cc41cebfae1b, disabling compactions & flushes 2024-12-04T12:48:34,732 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:34,732 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:34,732 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. after waiting 0 ms 2024-12-04T12:48:34,732 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:34,735 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/fd6a270c6969dabc7992cc41cebfae1b/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=6 2024-12-04T12:48:34,736 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:34,736 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:34,736 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fd6a270c6969dabc7992cc41cebfae1b: Waiting for close lock at 1733316514732Running coprocessor pre-close hooks at 1733316514732Disabling compacts and flushes for region at 1733316514732Disabling writes for close at 1733316514732Writing region close event to WAL at 1733316514732Running coprocessor post-close hooks at 1733316514736 (+4 ms)Closed at 1733316514736 2024-12-04T12:48:34,736 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,row0,1733316500251.fd6a270c6969dabc7992cc41cebfae1b. 2024-12-04T12:48:34,736 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 27f82bd5e2581f50581a7b2528369896, disabling compactions & flushes 2024-12-04T12:48:34,736 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:34,736 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:34,736 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. after waiting 0 ms 2024-12-04T12:48:34,736 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:34,740 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/table1/27f82bd5e2581f50581a7b2528369896/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=6 2024-12-04T12:48:34,741 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:34,741 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:34,741 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 27f82bd5e2581f50581a7b2528369896: Waiting for close lock at 1733316514736Running coprocessor pre-close hooks at 1733316514736Disabling compacts and flushes for region at 1733316514736Disabling writes for close at 1733316514736Writing region close event to WAL at 1733316514737 (+1 ms)Running coprocessor post-close hooks at 1733316514741 (+4 ms)Closed at 1733316514741 2024-12-04T12:48:34,741 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1733316500251.27f82bd5e2581f50581a7b2528369896. 2024-12-04T12:48:34,741 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 87e3e36a673c628af1440164c1fd3443, disabling compactions & flushes 2024-12-04T12:48:34,741 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:48:34,741 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:48:34,741 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. after waiting 0 ms 2024-12-04T12:48:34,741 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:48:34,741 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 87e3e36a673c628af1440164c1fd3443 3/3 column families, dataSize=378 B heapSize=1.63 KB 2024-12-04T12:48:34,751 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/.tmp/info/f5db1b4b534e47ae8c726f0b7a08b9ab is 159, key is ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a./info:regioninfo/1733316400610/Put/seqid=0 2024-12-04T12:48:34,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741996_1172 (size=19093) 2024-12-04T12:48:34,758 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/f2/6a9243ebf0554e99bcd2f969ee0cbd6f is 58, key is row-test-1733316396597-2-0/f2:q1/1733316471514/Put/seqid=0 2024-12-04T12:48:34,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741997_1173 (size=5464) 2024-12-04T12:48:34,928 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3ef1e15068edf0d0c9bdc810858b3ba2, 58f768c3c54039235f5a6360aacc8ebf, 87e3e36a673c628af1440164c1fd3443, 9dab1d763bdadc4adf3b254f1a913949, c988c84788a01c971b5789026625491a, deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:48:35,129 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3ef1e15068edf0d0c9bdc810858b3ba2, 58f768c3c54039235f5a6360aacc8ebf, 87e3e36a673c628af1440164c1fd3443, 9dab1d763bdadc4adf3b254f1a913949, c988c84788a01c971b5789026625491a, deae8c07fc7147a6340e830c87bd97da 2024-12-04T12:48:35,158 INFO [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.86 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/.tmp/info/f5db1b4b534e47ae8c726f0b7a08b9ab 2024-12-04T12:48:35,164 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=378 B at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/f2/6a9243ebf0554e99bcd2f969ee0cbd6f 2024-12-04T12:48:35,172 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/.tmp/f2/6a9243ebf0554e99bcd2f969ee0cbd6f as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f2/6a9243ebf0554e99bcd2f969ee0cbd6f 2024-12-04T12:48:35,173 INFO [regionserver/938b79ed23df:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T12:48:35,173 INFO [regionserver/938b79ed23df:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T12:48:35,178 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/f2/6a9243ebf0554e99bcd2f969ee0cbd6f, entries=7, sequenceid=348, filesize=5.3 K 2024-12-04T12:48:35,179 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 87e3e36a673c628af1440164c1fd3443 in 437ms, sequenceid=348, compaction requested=false 2024-12-04T12:48:35,184 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/.tmp/ns/12c976ac630a42dbaaa379f8579d7d4c is 85, key is ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8./ns:/1733316509707/DeleteFamily/seqid=0 2024-12-04T12:48:35,186 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/default/test-1733316396597/87e3e36a673c628af1440164c1fd3443/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=337 2024-12-04T12:48:35,187 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:35,187 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:48:35,187 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 87e3e36a673c628af1440164c1fd3443: Waiting for close lock at 1733316514741Running coprocessor pre-close hooks at 1733316514741Disabling compacts and flushes for region at 1733316514741Disabling writes for close at 1733316514741Obtaining lock to block concurrent updates at 1733316514741Preparing flush snapshotting stores in 87e3e36a673c628af1440164c1fd3443 at 1733316514741Finished memstore snapshotting test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443., syncing WAL and waiting on mvcc, flushsize=dataSize=378, getHeapSize=1616, getOffHeapSize=0, getCellsCount=7 at 1733316514742 (+1 ms)Flushing stores of test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. at 1733316514742Flushing 87e3e36a673c628af1440164c1fd3443/f2: creating writer at 1733316514742Flushing 87e3e36a673c628af1440164c1fd3443/f2: appending metadata at 1733316514758 (+16 ms)Flushing 87e3e36a673c628af1440164c1fd3443/f2: closing flushed file at 1733316514758Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2346b171: reopening flushed file at 1733316515170 (+412 ms)Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 87e3e36a673c628af1440164c1fd3443 in 437ms, sequenceid=348, compaction requested=false at 1733316515179 (+9 ms)Writing region close event to WAL at 1733316515183 (+4 ms)Running coprocessor post-close hooks at 1733316515187 (+4 ms)Closed at 1733316515187 2024-12-04T12:48:35,187 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1733316396597,,1733316397080.87e3e36a673c628af1440164c1fd3443. 2024-12-04T12:48:35,188 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3ef1e15068edf0d0c9bdc810858b3ba2, disabling compactions & flushes 2024-12-04T12:48:35,188 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2. 2024-12-04T12:48:35,188 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2. 2024-12-04T12:48:35,188 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2. after waiting 0 ms 2024-12-04T12:48:35,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741998_1174 (size=5760) 2024-12-04T12:48:35,188 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2. 2024-12-04T12:48:35,191 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/table2/3ef1e15068edf0d0c9bdc810858b3ba2/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-12-04T12:48:35,192 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:35,192 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2. 2024-12-04T12:48:35,192 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3ef1e15068edf0d0c9bdc810858b3ba2: Waiting for close lock at 1733316515188Running coprocessor pre-close hooks at 1733316515188Disabling compacts and flushes for region at 1733316515188Disabling writes for close at 1733316515188Writing region close event to WAL at 1733316515188Running coprocessor post-close hooks at 1733316515192 (+4 ms)Closed at 1733316515192 2024-12-04T12:48:35,192 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:table2,,1733316509706.3ef1e15068edf0d0c9bdc810858b3ba2. 2024-12-04T12:48:35,193 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c988c84788a01c971b5789026625491a, disabling compactions & flushes 2024-12-04T12:48:35,193 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:48:35,193 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:48:35,193 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. after waiting 0 ms 2024-12-04T12:48:35,193 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:48:35,198 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns2/test-17333163965971/c988c84788a01c971b5789026625491a/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2024-12-04T12:48:35,199 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:35,199 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:48:35,199 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c988c84788a01c971b5789026625491a: Waiting for close lock at 1733316515192Running coprocessor pre-close hooks at 1733316515193 (+1 ms)Disabling compacts and flushes for region at 1733316515193Disabling writes for close at 1733316515193Writing region close event to WAL at 1733316515195 (+2 ms)Running coprocessor post-close hooks at 1733316515199 (+4 ms)Closed at 1733316515199 2024-12-04T12:48:35,199 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17333163965971,,1733316399431.c988c84788a01c971b5789026625491a. 2024-12-04T12:48:35,199 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing deae8c07fc7147a6340e830c87bd97da, disabling compactions & flushes 2024-12-04T12:48:35,199 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. 2024-12-04T12:48:35,199 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. 2024-12-04T12:48:35,199 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. after waiting 0 ms 2024-12-04T12:48:35,199 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. 2024-12-04T12:48:35,202 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/ns4/test-17333163965973/deae8c07fc7147a6340e830c87bd97da/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-04T12:48:35,203 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:35,203 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. 2024-12-04T12:48:35,203 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for deae8c07fc7147a6340e830c87bd97da: Waiting for close lock at 1733316515199Running coprocessor pre-close hooks at 1733316515199Disabling compacts and flushes for region at 1733316515199Disabling writes for close at 1733316515199Writing region close event to WAL at 1733316515199Running coprocessor post-close hooks at 1733316515203 (+4 ms)Closed at 1733316515203 2024-12-04T12:48:35,203 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17333163965973,,1733316403925.deae8c07fc7147a6340e830c87bd97da. 2024-12-04T12:48:35,203 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9dab1d763bdadc4adf3b254f1a913949, disabling compactions & flushes 2024-12-04T12:48:35,203 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:48:35,203 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:48:35,203 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. after waiting 0 ms 2024-12-04T12:48:35,203 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:48:35,203 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9dab1d763bdadc4adf3b254f1a913949 2/2 column families, dataSize=2.12 KB heapSize=3.43 KB 2024-12-04T12:48:35,207 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/meta/85fe2b69705a4a268ab3605f31e17439 is 158, key is trslm:hdfs://localhost:41555/backupUT\x00ns2:test-17333163965971/meta:log-roll-map/1733316489065/Put/seqid=0 2024-12-04T12:48:35,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741999_1175 (size=6208) 2024-12-04T12:48:35,329 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 58f768c3c54039235f5a6360aacc8ebf, 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:48:35,529 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 58f768c3c54039235f5a6360aacc8ebf, 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:48:35,590 INFO [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=474 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/.tmp/ns/12c976ac630a42dbaaa379f8579d7d4c 2024-12-04T12:48:35,611 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=944 B at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/meta/85fe2b69705a4a268ab3605f31e17439 2024-12-04T12:48:35,618 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/.tmp/rep_barrier/26d6bac626684d6bb36ac27e079a877c is 94, key is ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8./rep_barrier:/1733316509707/DeleteFamily/seqid=0 2024-12-04T12:48:35,618 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/session/fbc3447374764e029a9683f36997f9d4 is 588, key is session:backup_1733316476524/session:context/1733316489509/Put/seqid=0 2024-12-04T12:48:35,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073742000_1176 (size=5640) 2024-12-04T12:48:35,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073742001_1177 (size=5689) 2024-12-04T12:48:35,730 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-04T12:48:35,730 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 9dab1d763bdadc4adf3b254f1a913949=backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., 58f768c3c54039235f5a6360aacc8ebf=backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf.} 2024-12-04T12:48:35,730 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 58f768c3c54039235f5a6360aacc8ebf, 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:48:35,930 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 58f768c3c54039235f5a6360aacc8ebf, 9dab1d763bdadc4adf3b254f1a913949 2024-12-04T12:48:36,024 INFO [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=266 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/.tmp/rep_barrier/26d6bac626684d6bb36ac27e079a877c 2024-12-04T12:48:36,026 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.20 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/session/fbc3447374764e029a9683f36997f9d4 2024-12-04T12:48:36,034 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/meta/85fe2b69705a4a268ab3605f31e17439 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/85fe2b69705a4a268ab3605f31e17439 2024-12-04T12:48:36,039 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/meta/85fe2b69705a4a268ab3605f31e17439, entries=7, sequenceid=52, filesize=6.1 K 2024-12-04T12:48:36,040 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/.tmp/session/fbc3447374764e029a9683f36997f9d4 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/fbc3447374764e029a9683f36997f9d4 2024-12-04T12:48:36,045 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/session/fbc3447374764e029a9683f36997f9d4, entries=2, sequenceid=52, filesize=5.6 K 2024-12-04T12:48:36,046 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~2.12 KB/2170, heapSize ~3.40 KB/3480, currentSize=0 B/0 for 9dab1d763bdadc4adf3b254f1a913949 in 842ms, sequenceid=52, compaction requested=true 2024-12-04T12:48:36,048 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/.tmp/table/c0e412d23acf4d30bf4d75919eb4a1cf is 88, key is ns2:table2,,1733316495261.df2cfadd8eadf907300286d81ee0cbf8./table:/1733316509707/DeleteFamily/seqid=0 2024-12-04T12:48:36,049 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system/9dab1d763bdadc4adf3b254f1a913949/recovered.edits/55.seqid, newMaxSeqId=55, maxSeqId=1 2024-12-04T12:48:36,050 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:36,050 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:48:36,050 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9dab1d763bdadc4adf3b254f1a913949: Waiting for close lock at 1733316515203Running coprocessor pre-close hooks at 1733316515203Disabling compacts and flushes for region at 1733316515203Disabling writes for close at 1733316515203Obtaining lock to block concurrent updates at 1733316515203Preparing flush snapshotting stores in 9dab1d763bdadc4adf3b254f1a913949 at 1733316515203Finished memstore snapshotting backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949., syncing WAL and waiting on mvcc, flushsize=dataSize=2170, getHeapSize=3480, getOffHeapSize=0, getCellsCount=11 at 1733316515203Flushing stores of backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. at 1733316515204 (+1 ms)Flushing 9dab1d763bdadc4adf3b254f1a913949/meta: creating writer at 1733316515204Flushing 9dab1d763bdadc4adf3b254f1a913949/meta: appending metadata at 1733316515206 (+2 ms)Flushing 9dab1d763bdadc4adf3b254f1a913949/meta: closing flushed file at 1733316515207 (+1 ms)Flushing 9dab1d763bdadc4adf3b254f1a913949/session: creating writer at 1733316515616 (+409 ms)Flushing 9dab1d763bdadc4adf3b254f1a913949/session: appending metadata at 1733316515618 (+2 ms)Flushing 9dab1d763bdadc4adf3b254f1a913949/session: closing flushed file at 1733316515618Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45801a44: reopening flushed file at 1733316516033 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74eca881: reopening flushed file at 1733316516039 (+6 ms)Finished flush of dataSize ~2.12 KB/2170, heapSize ~3.40 KB/3480, currentSize=0 B/0 for 9dab1d763bdadc4adf3b254f1a913949 in 842ms, sequenceid=52, compaction requested=true at 1733316516046 (+7 ms)Writing region close event to WAL at 1733316516046Running coprocessor post-close hooks at 1733316516050 (+4 ms)Closed at 1733316516050 2024-12-04T12:48:36,050 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1733316406900.9dab1d763bdadc4adf3b254f1a913949. 2024-12-04T12:48:36,050 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 58f768c3c54039235f5a6360aacc8ebf, disabling compactions & flushes 2024-12-04T12:48:36,051 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. 2024-12-04T12:48:36,051 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. 2024-12-04T12:48:36,051 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. after waiting 0 ms 2024-12-04T12:48:36,051 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. 2024-12-04T12:48:36,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073742002_1178 (size=6364) 2024-12-04T12:48:36,054 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/backup/system_bulk/58f768c3c54039235f5a6360aacc8ebf/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-04T12:48:36,055 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:36,055 INFO [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. 2024-12-04T12:48:36,055 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 58f768c3c54039235f5a6360aacc8ebf: Waiting for close lock at 1733316516050Running coprocessor pre-close hooks at 1733316516050Disabling compacts and flushes for region at 1733316516050Disabling writes for close at 1733316516051 (+1 ms)Writing region close event to WAL at 1733316516051Running coprocessor post-close hooks at 1733316516055 (+4 ms)Closed at 1733316516055 2024-12-04T12:48:36,055 DEBUG [RS_CLOSE_REGION-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1733316409084.58f768c3c54039235f5a6360aacc8ebf. 2024-12-04T12:48:36,131 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T12:48:36,331 DEBUG [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T12:48:36,454 INFO [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.39 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/.tmp/table/c0e412d23acf4d30bf4d75919eb4a1cf 2024-12-04T12:48:36,465 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/.tmp/info/f5db1b4b534e47ae8c726f0b7a08b9ab as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/info/f5db1b4b534e47ae8c726f0b7a08b9ab 2024-12-04T12:48:36,470 INFO [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/info/f5db1b4b534e47ae8c726f0b7a08b9ab, entries=114, sequenceid=97, filesize=18.6 K 2024-12-04T12:48:36,471 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/.tmp/ns/12c976ac630a42dbaaa379f8579d7d4c as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/ns/12c976ac630a42dbaaa379f8579d7d4c 2024-12-04T12:48:36,476 INFO [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/ns/12c976ac630a42dbaaa379f8579d7d4c, entries=10, sequenceid=97, filesize=5.6 K 2024-12-04T12:48:36,477 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/.tmp/rep_barrier/26d6bac626684d6bb36ac27e079a877c as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/rep_barrier/26d6bac626684d6bb36ac27e079a877c 2024-12-04T12:48:36,482 INFO [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/rep_barrier/26d6bac626684d6bb36ac27e079a877c, entries=3, sequenceid=97, filesize=5.5 K 2024-12-04T12:48:36,483 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/.tmp/table/c0e412d23acf4d30bf4d75919eb4a1cf as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/table/c0e412d23acf4d30bf4d75919eb4a1cf 2024-12-04T12:48:36,488 INFO [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/table/c0e412d23acf4d30bf4d75919eb4a1cf, entries=21, sequenceid=97, filesize=6.2 K 2024-12-04T12:48:36,489 INFO [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~23.97 KB/24544, heapSize ~42.34 KB/43352, currentSize=0 B/0 for 1588230740 in 1761ms, sequenceid=97, compaction requested=false 2024-12-04T12:48:36,494 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/data/hbase/meta/1588230740/recovered.edits/100.seqid, newMaxSeqId=100, maxSeqId=1 2024-12-04T12:48:36,495 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:36,495 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T12:48:36,495 INFO [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T12:48:36,495 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733316514728Running coprocessor pre-close hooks at 1733316514728Disabling compacts and flushes for region at 1733316514728Disabling writes for close at 1733316514728Obtaining lock to block concurrent updates at 1733316514728Preparing flush snapshotting stores in 1588230740 at 1733316514728Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=24544, getHeapSize=43352, getOffHeapSize=0, getCellsCount=235 at 1733316514728Flushing stores of hbase:meta,,1.1588230740 at 1733316514729 (+1 ms)Flushing 1588230740/info: creating writer at 1733316514729Flushing 1588230740/info: appending metadata at 1733316514751 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733316514751Flushing 1588230740/ns: creating writer at 1733316515168 (+417 ms)Flushing 1588230740/ns: appending metadata at 1733316515184 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733316515184Flushing 1588230740/rep_barrier: creating writer at 1733316515602 (+418 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733316515617 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733316515617Flushing 1588230740/table: creating writer at 1733316516033 (+416 ms)Flushing 1588230740/table: appending metadata at 1733316516048 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733316516048Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ad01763: reopening flushed file at 1733316516464 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52191390: reopening flushed file at 1733316516470 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c2f6138: reopening flushed file at 1733316516476 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a7a0aef: reopening flushed file at 1733316516482 (+6 ms)Finished flush of dataSize ~23.97 KB/24544, heapSize ~42.34 KB/43352, currentSize=0 B/0 for 1588230740 in 1761ms, sequenceid=97, compaction requested=false at 1733316516489 (+7 ms)Writing region close event to WAL at 1733316516491 (+2 ms)Running coprocessor post-close hooks at 1733316516495 (+4 ms)Closed at 1733316516495 2024-12-04T12:48:36,495 DEBUG [RS_CLOSE_META-regionserver/938b79ed23df:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T12:48:36,531 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(976): stopping server 938b79ed23df,38879,1733316387992; all regions closed. 2024-12-04T12:48:36,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741955_1131 (size=16160) 2024-12-04T12:48:36,534 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.meta.1733316481006.meta not finished, retry = 0 2024-12-04T12:48:36,641 DEBUG [RS:0;938b79ed23df:38879 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs 2024-12-04T12:48:36,641 INFO [RS:0;938b79ed23df:38879 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 938b79ed23df%2C38879%2C1733316387992.meta:.meta(num 1733316481006) 2024-12-04T12:48:36,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741954_1130 (size=4448) 2024-12-04T12:48:36,646 DEBUG [RS:0;938b79ed23df:38879 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs 2024-12-04T12:48:36,646 INFO [RS:0;938b79ed23df:38879 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-0:(num 1733316480990) 2024-12-04T12:48:36,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741956_1132 (size=7600) 2024-12-04T12:48:36,649 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/WALs/938b79ed23df,38879,1733316387992/938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1.1733316481020 not finished, retry = 0 2024-12-04T12:48:36,759 DEBUG [RS:0;938b79ed23df:38879 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/oldWALs 2024-12-04T12:48:36,759 INFO [RS:0;938b79ed23df:38879 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 938b79ed23df%2C38879%2C1733316387992.938b79ed23df%2C38879%2C1733316387992.regiongroup-1:(num 1733316481020) 2024-12-04T12:48:36,759 DEBUG [RS:0;938b79ed23df:38879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:36,759 INFO [RS:0;938b79ed23df:38879 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T12:48:36,759 INFO [RS:0;938b79ed23df:38879 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T12:48:36,759 INFO [RS:0;938b79ed23df:38879 {}] hbase.ChoreService(370): Chore service for: regionserver/938b79ed23df:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T12:48:36,759 INFO [RS:0;938b79ed23df:38879 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T12:48:36,759 INFO [regionserver/938b79ed23df:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T12:48:36,760 INFO [RS:0;938b79ed23df:38879 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38879 2024-12-04T12:48:36,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/938b79ed23df,38879,1733316387992 2024-12-04T12:48:36,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-12-04T12:48:36,779 INFO [RS:0;938b79ed23df:38879 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T12:48:36,787 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [938b79ed23df,38879,1733316387992] 2024-12-04T12:48:36,795 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /1/draining/938b79ed23df,38879,1733316387992 already deleted, retry=false 2024-12-04T12:48:36,795 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 938b79ed23df,38879,1733316387992 expired; onlineServers=0 2024-12-04T12:48:36,795 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '938b79ed23df,36049,1733316386472' ***** 2024-12-04T12:48:36,795 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T12:48:36,795 INFO [M:0;938b79ed23df:36049 {}] hbase.HBaseServerBase(468): Stop info server 2024-12-04T12:48:36,801 INFO [M:0;938b79ed23df:36049 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@3bb1047d{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-12-04T12:48:36,801 INFO [M:0;938b79ed23df:36049 {}] server.AbstractConnector(383): Stopped ServerConnector@678f6157{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-12-04T12:48:36,801 INFO [M:0;938b79ed23df:36049 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T12:48:36,802 INFO [M:0;938b79ed23df:36049 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@390508af{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-12-04T12:48:36,802 INFO [M:0;938b79ed23df:36049 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3763b03b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,STOPPED} 2024-12-04T12:48:36,802 INFO [M:0;938b79ed23df:36049 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T12:48:36,802 INFO [M:0;938b79ed23df:36049 {}] master.HMaster(1770): Stopping master jetty server 2024-12-04T12:48:36,803 INFO [M:0;938b79ed23df:36049 {}] server.AbstractConnector(383): Stopped ServerConnector@6df710df{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-12-04T12:48:36,803 INFO [M:0;938b79ed23df:36049 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T12:48:36,803 DEBUG [M:0;938b79ed23df:36049 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T12:48:36,803 INFO [M:0;938b79ed23df:36049 {}] master.BackupLogCleaner(194): Stopping BackupLogCleaner 2024-12-04T12:48:36,803 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T12:48:36,803 INFO [M:0;938b79ed23df:36049 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;938b79ed23df:36049. 2024-12-04T12:48:36,803 DEBUG [master/938b79ed23df:0:becomeActiveMaster-HFileCleaner.large.0-1733316390962 {}] cleaner.HFileCleaner(306): Exit Thread[master/938b79ed23df:0:becomeActiveMaster-HFileCleaner.large.0-1733316390962,5,FailOnTimeoutGroup] 2024-12-04T12:48:36,803 DEBUG [M:0;938b79ed23df:36049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupHFileCleaner.stop(BackupHFileCleaner.java:156) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.cleanup(CleanerChore.java:444) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.cleanup(HFileCleaner.java:239) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:333) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:322) at org.apache.hadoop.hbase.master.HMaster.stopChores(HMaster.java:1923) at org.apache.hadoop.hbase.HBaseServerBase.stopChoreService(HBaseServerBase.java:439) at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1777) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T12:48:36,803 DEBUG [M:0;938b79ed23df:36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:36,803 DEBUG [M:0;938b79ed23df:36049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T12:48:36,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2024-12-04T12:48:36,803 DEBUG [master/938b79ed23df:0:becomeActiveMaster-HFileCleaner.small.0-1733316390963 {}] cleaner.HFileCleaner(306): Exit Thread[master/938b79ed23df:0:becomeActiveMaster-HFileCleaner.small.0-1733316390963,5,FailOnTimeoutGroup] 2024-12-04T12:48:36,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T12:48:36,804 DEBUG [M:0;938b79ed23df:36049 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T12:48:36,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T12:48:36,804 INFO [M:0;938b79ed23df:36049 {}] hbase.ChoreService(370): Chore service for: master/938b79ed23df:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T12:48:36,804 INFO [M:0;938b79ed23df:36049 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T12:48:36,804 DEBUG [M:0;938b79ed23df:36049 {}] master.HMaster(1795): Stopping service threads 2024-12-04T12:48:36,804 INFO [M:0;938b79ed23df:36049 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T12:48:36,804 INFO [M:0;938b79ed23df:36049 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T12:48:36,804 ERROR [M:0;938b79ed23df:36049 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-1,5,PEWorkerGroup] Thread[HFileArchiver-2,5,PEWorkerGroup] Thread[HFileArchiver-3,5,PEWorkerGroup] 2024-12-04T12:48:36,805 INFO [M:0;938b79ed23df:36049 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T12:48:36,805 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T12:48:36,805 DEBUG [M:0;938b79ed23df:36049 {}] zookeeper.ZKUtil(347): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2024-12-04T12:48:36,805 WARN [M:0;938b79ed23df:36049 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T12:48:36,806 INFO [M:0;938b79ed23df:36049 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/.lastflushedseqids 2024-12-04T12:48:36,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073742003_1179 (size=761) 2024-12-04T12:48:36,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T12:48:36,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17333163965972 2024-12-04T12:48:36,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-12-04T12:48:36,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2024-12-04T12:48:36,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T12:48:36,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-12-04T12:48:36,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17333163965971 2024-12-04T12:48:36,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1733316396597 2024-12-04T12:48:36,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17333163965973 2024-12-04T12:48:36,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T12:48:36,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2024-12-04T12:48:36,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T12:48:36,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38879-0x101a18ee11e0001, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T12:48:36,888 INFO [RS:0;938b79ed23df:38879 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T12:48:36,888 INFO [RS:0;938b79ed23df:38879 {}] regionserver.HRegionServer(1031): Exiting; stopping=938b79ed23df,38879,1733316387992; zookeeper connection closed. 2024-12-04T12:48:36,889 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3166944e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3166944e 2024-12-04T12:48:36,889 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T12:48:37,215 INFO [M:0;938b79ed23df:36049 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T12:48:37,216 INFO [M:0;938b79ed23df:36049 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T12:48:37,216 DEBUG [M:0;938b79ed23df:36049 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T12:48:37,226 INFO [M:0;938b79ed23df:36049 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T12:48:37,226 DEBUG [M:0;938b79ed23df:36049 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T12:48:37,227 DEBUG [M:0;938b79ed23df:36049 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-12-04T12:48:37,227 DEBUG [M:0;938b79ed23df:36049 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T12:48:37,227 INFO [M:0;938b79ed23df:36049 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=441.45 KB heapSize=518.45 KB 2024-12-04T12:48:37,240 DEBUG [M:0;938b79ed23df:36049 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e0527e59a9e545058d8a9f41eaee9b12 is 82, key is hbase:meta,,1/info:regioninfo/1733316392257/Put/seqid=0 2024-12-04T12:48:37,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073742004_1180 (size=5672) 2024-12-04T12:48:37,646 INFO [M:0;938b79ed23df:36049 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e0527e59a9e545058d8a9f41eaee9b12 2024-12-04T12:48:37,675 DEBUG [M:0;938b79ed23df:36049 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8307197f641848ba9e9cbf8ba183c98d is 2201, key is \x00\x00\x00\x00\x00\x00\x00Q/proc:d/1733316506925/Put/seqid=0 2024-12-04T12:48:37,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073742005_1181 (size=30170) 2024-12-04T12:48:38,082 INFO [M:0;938b79ed23df:36049 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=440.90 KB at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8307197f641848ba9e9cbf8ba183c98d 2024-12-04T12:48:38,093 INFO [M:0;938b79ed23df:36049 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8307197f641848ba9e9cbf8ba183c98d 2024-12-04T12:48:38,108 DEBUG [M:0;938b79ed23df:36049 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/75bd9da1b91d45d19ca57802b4471689 is 69, key is 938b79ed23df,38879,1733316387992/rs:state/1733316391086/Put/seqid=0 2024-12-04T12:48:38,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073742006_1182 (size=5156) 2024-12-04T12:48:38,513 INFO [M:0;938b79ed23df:36049 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/75bd9da1b91d45d19ca57802b4471689 2024-12-04T12:48:38,525 DEBUG [M:0;938b79ed23df:36049 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e0527e59a9e545058d8a9f41eaee9b12 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e0527e59a9e545058d8a9f41eaee9b12 2024-12-04T12:48:38,530 INFO [M:0;938b79ed23df:36049 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e0527e59a9e545058d8a9f41eaee9b12, entries=8, sequenceid=908, filesize=5.5 K 2024-12-04T12:48:38,531 DEBUG [M:0;938b79ed23df:36049 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8307197f641848ba9e9cbf8ba183c98d as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8307197f641848ba9e9cbf8ba183c98d 2024-12-04T12:48:38,535 INFO [M:0;938b79ed23df:36049 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8307197f641848ba9e9cbf8ba183c98d 2024-12-04T12:48:38,535 INFO [M:0;938b79ed23df:36049 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8307197f641848ba9e9cbf8ba183c98d, entries=94, sequenceid=908, filesize=29.5 K 2024-12-04T12:48:38,536 DEBUG [M:0;938b79ed23df:36049 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/75bd9da1b91d45d19ca57802b4471689 as hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/75bd9da1b91d45d19ca57802b4471689 2024-12-04T12:48:38,540 INFO [M:0;938b79ed23df:36049 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41555/user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/75bd9da1b91d45d19ca57802b4471689, entries=1, sequenceid=908, filesize=5.0 K 2024-12-04T12:48:38,541 INFO [M:0;938b79ed23df:36049 {}] regionserver.HRegion(3140): Finished flush of dataSize ~441.45 KB/452047, heapSize ~518.15 KB/530584, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1314ms, sequenceid=908, compaction requested=false 2024-12-04T12:48:38,542 INFO [M:0;938b79ed23df:36049 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T12:48:38,543 DEBUG [M:0;938b79ed23df:36049 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733316517216Disabling compacts and flushes for region at 1733316517216Disabling writes for close at 1733316517227 (+11 ms)Obtaining lock to block concurrent updates at 1733316517227Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733316517227Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=452047, getHeapSize=530824, getOffHeapSize=0, getCellsCount=1031 at 1733316517227Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733316517228 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733316517228Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733316517240 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733316517240Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733316517658 (+418 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733316517675 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733316517675Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733316518093 (+418 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733316518107 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733316518107Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b0f84e4: reopening flushed file at 1733316518524 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65358171: reopening flushed file at 1733316518530 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29b3e5b7: reopening flushed file at 1733316518535 (+5 ms)Finished flush of dataSize ~441.45 KB/452047, heapSize ~518.15 KB/530584, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1314ms, sequenceid=908, compaction requested=false at 1733316518541 (+6 ms)Writing region close event to WAL at 1733316518542 (+1 ms)Closed at 1733316518542 2024-12-04T12:48:38,545 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d86d33b8-65d5-7bc8-c21b-a9314ba57b8a/MasterData/WALs/938b79ed23df,36049,1733316386472/938b79ed23df%2C36049%2C1733316386472.938b79ed23df%2C36049%2C1733316386472.regiongroup-0.1733316390182 not finished, retry = 0 2024-12-04T12:48:38,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44611 is added to blk_1073741830_1006 (size=511978) 2024-12-04T12:48:38,647 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T12:48:38,647 INFO [M:0;938b79ed23df:36049 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T12:48:38,647 INFO [M:0;938b79ed23df:36049 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2024-12-04T12:48:38,647 INFO [M:0;938b79ed23df:36049 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36049 2024-12-04T12:48:38,647 INFO [M:0;938b79ed23df:36049 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T12:48:38,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T12:48:38,795 INFO [M:0;938b79ed23df:36049 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T12:48:38,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36049-0x101a18ee11e0000, quorum=127.0.0.1:60209, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T12:48:38,819 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d3c17d4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T12:48:38,822 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@575d15e0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T12:48:38,823 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T12:48:38,823 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@308dcebb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T12:48:38,823 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@347c3eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,STOPPED} 2024-12-04T12:48:38,825 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T12:48:38,825 WARN [BP-117991357-172.17.0.3-1733316383096 heartbeating to localhost/127.0.0.1:41555 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T12:48:38,825 WARN [BP-117991357-172.17.0.3-1733316383096 heartbeating to localhost/127.0.0.1:41555 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-117991357-172.17.0.3-1733316383096 (Datanode Uuid 74cbd0c3-db18-4c04-abbd-e9560e582016) service to localhost/127.0.0.1:41555 2024-12-04T12:48:38,825 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T12:48:38,827 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/cluster_4dfbfc9e-7777-d399-4020-07547724c305/data/data1/current/BP-117991357-172.17.0.3-1733316383096 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T12:48:38,827 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/cluster_4dfbfc9e-7777-d399-4020-07547724c305/data/data2/current/BP-117991357-172.17.0.3-1733316383096 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T12:48:38,827 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T12:48:38,832 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1d57f2ce{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T12:48:38,833 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@382e7786{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T12:48:38,833 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T12:48:38,833 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@325f8c88{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T12:48:38,833 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@205baae5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,STOPPED} 2024-12-04T12:48:38,840 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T12:48:38,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T12:48:38,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2377): Stopping mini mapreduce cluster... 2024-12-04T12:48:38,901 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4a556c09{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-04T12:48:38,902 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c619d4c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T12:48:38,902 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T12:48:38,902 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1edcf525{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-04T12:48:38,902 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@625a811e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,STOPPED} 2024-12-04T12:48:38,910 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3860df07{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-04T12:48:38,910 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1ed8df47{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T12:48:38,910 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T12:48:38,910 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6c8395ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-04T12:48:38,910 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@51553806{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,STOPPED} 2024-12-04T12:48:38,917 ERROR [Thread[Thread-270,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-12-04T12:48:38,917 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@395940ac{cluster,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-12-04T12:48:38,917 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3f7727e1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T12:48:38,918 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T12:48:38,918 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336bed82{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-04T12:48:38,918 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ef40a45{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,STOPPED} 2024-12-04T12:48:38,920 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2024-12-04T12:48:38,922 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(72): Returning, interrupted : java.lang.InterruptedException 2024-12-04T12:48:38,922 ERROR [ResourceManager Event Processor Monitor {}] resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessorMonitor(1193): Returning, interrupted : java.lang.InterruptedException: sleep interrupted 2024-12-04T12:48:38,923 ERROR [Thread[Thread-293,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-12-04T12:48:38,927 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@50e4b420{jobhistory,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-12-04T12:48:38,928 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@417ea536{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T12:48:38,928 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T12:48:38,928 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75c0ec4a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-04T12:48:38,928 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a8eed8e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/cded3315-5be1-2d9a-8654-431de1a34354/hadoop.log.dir/,STOPPED} 2024-12-04T12:48:38,929 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-12-04T12:48:38,929 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2380): Mini mapreduce cluster stopped