2024-11-12 17:33:44,285 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-12 17:33:44,301 main DEBUG Took 0.014105 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 17:33:44,302 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 17:33:44,302 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 17:33:44,303 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 17:33:44,305 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 17:33:44,319 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 17:33:44,349 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 17:33:44,351 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 17:33:44,351 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 17:33:44,352 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 17:33:44,352 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 17:33:44,353 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 17:33:44,354 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 17:33:44,354 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 17:33:44,355 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 17:33:44,355 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 17:33:44,356 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 17:33:44,356 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 17:33:44,357 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 17:33:44,358 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 17:33:44,359 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 17:33:44,359 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 17:33:44,360 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 17:33:44,360 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 17:33:44,361 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 17:33:44,361 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 17:33:44,362 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 17:33:44,363 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 17:33:44,363 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 17:33:44,364 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 17:33:44,365 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 17:33:44,365 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 17:33:44,367 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 17:33:44,369 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 17:33:44,371 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 17:33:44,372 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 17:33:44,374 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 17:33:44,374 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 17:33:44,387 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 17:33:44,391 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 17:33:44,395 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 17:33:44,396 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 17:33:44,397 main DEBUG createAppenders(={Console}) 2024-11-12 17:33:44,398 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 initialized 2024-11-12 17:33:44,398 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-12 17:33:44,398 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 OK. 2024-11-12 17:33:44,399 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 17:33:44,400 main DEBUG OutputStream closed 2024-11-12 17:33:44,400 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 17:33:44,401 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 17:33:44,401 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3c73951 OK 2024-11-12 17:33:44,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 17:33:44,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 17:33:44,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 17:33:44,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 17:33:44,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 17:33:44,516 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 17:33:44,516 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 17:33:44,517 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 17:33:44,517 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 17:33:44,518 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 17:33:44,518 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 17:33:44,519 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 17:33:44,519 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 17:33:44,520 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 17:33:44,528 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 17:33:44,529 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 17:33:44,529 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 17:33:44,530 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 17:33:44,535 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 17:33:44,536 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@61fe30) with optional ClassLoader: null 2024-11-12 17:33:44,537 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 17:33:44,538 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@61fe30] started OK. 2024-11-12T17:33:44,613 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackup timeout: 13 mins 2024-11-12 17:33:44,617 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 17:33:44,617 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T17:33:45,200 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d 2024-11-12T17:33:45,206 DEBUG [Time-limited test {}] impl.BackupManager(124): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-12T17:33:45,209 DEBUG [Time-limited test {}] impl.BackupManager(154): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T17:33:45,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T17:33:45,231 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/cluster_5efe2478-d756-86b3-13f2-642347e6bab0, deleteOnExit=true 2024-11-12T17:33:45,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-12T17:33:45,233 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/test.cache.data in system properties and HBase conf 2024-11-12T17:33:45,234 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T17:33:45,234 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir in system properties and HBase conf 2024-11-12T17:33:45,235 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T17:33:45,236 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T17:33:45,236 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T17:33:45,373 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T17:33:45,558 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T17:33:45,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T17:33:45,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T17:33:45,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T17:33:45,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T17:33:45,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T17:33:45,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T17:33:45,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T17:33:45,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T17:33:45,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T17:33:45,571 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/nfs.dump.dir in system properties and HBase conf 2024-11-12T17:33:45,571 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/java.io.tmpdir in system properties and HBase conf 2024-11-12T17:33:45,572 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T17:33:45,572 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T17:33:45,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T17:33:46,706 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T17:33:46,805 INFO [Time-limited test {}] log.Log(170): Logging initialized @3813ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T17:33:46,995 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T17:33:47,128 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T17:33:47,178 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T17:33:47,178 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T17:33:47,181 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T17:33:47,202 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T17:33:47,208 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71f46724{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,AVAILABLE} 2024-11-12T17:33:47,210 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17280962{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T17:33:47,443 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6df1a11{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/java.io.tmpdir/jetty-localhost-38857-hadoop-hdfs-3_4_1-tests_jar-_-any-646740794931166924/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T17:33:47,453 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34c8bc74{HTTP/1.1, (http/1.1)}{localhost:38857} 2024-11-12T17:33:47,454 INFO [Time-limited test {}] server.Server(415): Started @4463ms 2024-11-12T17:33:47,902 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T17:33:47,910 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T17:33:47,912 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T17:33:47,913 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T17:33:47,913 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T17:33:47,915 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@9b2b78d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,AVAILABLE} 2024-11-12T17:33:47,916 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@19fb95db{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T17:33:48,066 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1dce7e65{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/java.io.tmpdir/jetty-localhost-36485-hadoop-hdfs-3_4_1-tests_jar-_-any-1176183449781431739/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T17:33:48,067 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2044050{HTTP/1.1, (http/1.1)}{localhost:36485} 2024-11-12T17:33:48,071 INFO [Time-limited test {}] server.Server(415): Started @5080ms 2024-11-12T17:33:48,140 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T17:33:48,899 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/cluster_5efe2478-d756-86b3-13f2-642347e6bab0/data/data1/current/BP-2101155470-172.17.0.2-1731432826388/current, will proceed with Du for space computation calculation, 2024-11-12T17:33:48,901 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/cluster_5efe2478-d756-86b3-13f2-642347e6bab0/data/data2/current/BP-2101155470-172.17.0.2-1731432826388/current, will proceed with Du for space computation calculation, 2024-11-12T17:33:48,959 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T17:33:49,044 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa2e814eb8e7c476f with lease ID 0x4e8cdee4809b363d: Processing first storage report for DS-598c5b73-e17e-430f-b948-145aa5f7a106 from datanode DatanodeRegistration(127.0.0.1:41947, datanodeUuid=b7f84aa1-4646-4913-8f3f-fdf3fcbd413d, infoPort=41645, infoSecurePort=0, ipcPort=40705, storageInfo=lv=-57;cid=testClusterID;nsid=1284240676;c=1731432826388) 2024-11-12T17:33:49,046 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa2e814eb8e7c476f with lease ID 0x4e8cdee4809b363d: from storage DS-598c5b73-e17e-430f-b948-145aa5f7a106 node DatanodeRegistration(127.0.0.1:41947, datanodeUuid=b7f84aa1-4646-4913-8f3f-fdf3fcbd413d, infoPort=41645, infoSecurePort=0, ipcPort=40705, storageInfo=lv=-57;cid=testClusterID;nsid=1284240676;c=1731432826388), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-12T17:33:49,046 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa2e814eb8e7c476f with lease ID 0x4e8cdee4809b363d: Processing first storage report for DS-f6cf06bd-55cd-4c00-86ef-13a22e56fa6a from datanode DatanodeRegistration(127.0.0.1:41947, datanodeUuid=b7f84aa1-4646-4913-8f3f-fdf3fcbd413d, infoPort=41645, infoSecurePort=0, ipcPort=40705, storageInfo=lv=-57;cid=testClusterID;nsid=1284240676;c=1731432826388) 2024-11-12T17:33:49,047 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa2e814eb8e7c476f with lease ID 0x4e8cdee4809b363d: from storage DS-f6cf06bd-55cd-4c00-86ef-13a22e56fa6a node DatanodeRegistration(127.0.0.1:41947, datanodeUuid=b7f84aa1-4646-4913-8f3f-fdf3fcbd413d, infoPort=41645, infoSecurePort=0, ipcPort=40705, storageInfo=lv=-57;cid=testClusterID;nsid=1284240676;c=1731432826388), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T17:33:49,075 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d 2024-11-12T17:33:49,216 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/cluster_5efe2478-d756-86b3-13f2-642347e6bab0/zookeeper_0, clientPort=61065, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/cluster_5efe2478-d756-86b3-13f2-642347e6bab0/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/cluster_5efe2478-d756-86b3-13f2-642347e6bab0/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T17:33:49,238 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61065 2024-11-12T17:33:49,256 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T17:33:49,260 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T17:33:49,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741825_1001 (size=7) 2024-11-12T17:33:50,000 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 with version=8 2024-11-12T17:33:50,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/hbase-staging 2024-11-12T17:33:50,018 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(136): Setting Master Port to random. 2024-11-12T17:33:50,018 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(143): Setting RegionServer Port to random. 2024-11-12T17:33:50,018 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(153): Setting RS InfoServer Port to random. 2024-11-12T17:33:50,018 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(161): Setting Master InfoServer Port to random. 2024-11-12T17:33:50,148 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T17:33:50,453 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1f32af3374c6:0 server-side Connection retries=45 2024-11-12T17:33:50,466 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T17:33:50,466 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T17:33:50,471 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T17:33:50,472 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T17:33:50,472 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T17:33:50,625 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T17:33:50,702 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T17:33:50,715 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T17:33:50,721 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T17:33:50,756 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 23342 (auto-detected) 2024-11-12T17:33:50,758 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-12T17:33:50,782 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39475 2024-11-12T17:33:50,812 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39475 connecting to ZooKeeper ensemble=127.0.0.1:61065 2024-11-12T17:33:50,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:394750x0, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T17:33:50,864 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39475-0x1019d8bc6660000 connected 2024-11-12T17:33:50,913 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T17:33:50,917 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T17:33:50,930 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-12T17:33:50,963 INFO [Time-limited test {}] log.Log(170): Logging initialized @7972ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T17:33:51,030 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-12T17:33:51,030 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-12T17:33:51,032 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-12T17:33:51,032 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-12T17:33:51,033 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-12T17:33:51,036 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2024-11-12T17:33:51,036 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-12T17:33:51,036 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-12T17:33:51,039 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-12T17:33:51,127 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 41353 2024-11-12T17:33:51,129 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-12T17:33:51,147 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-12T17:33:51,148 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@7ca98a11{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,AVAILABLE} 2024-11-12T17:33:51,149 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-12T17:33:51,149 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5b460d1c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-12T17:33:51,192 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-12T17:33:51,200 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T17:33:51,200 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T17:33:51,201 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T17:33:51,205 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-12T17:33:51,928 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@56e6bf8{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/master} 2024-11-12T17:33:51,933 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@69cb2ebd{HTTP/1.1, (http/1.1)}{0.0.0.0:41353} 2024-11-12T17:33:51,934 INFO [Time-limited test {}] server.Server(415): Started @8943ms 2024-11-12T17:33:51,939 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6, hbase.cluster.distributed=false 2024-11-12T17:33:51,969 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-12T17:33:51,974 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39475 2024-11-12T17:33:51,974 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39475 2024-11-12T17:33:51,975 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39475 2024-11-12T17:33:51,975 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39475 2024-11-12T17:33:51,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39475 2024-11-12T17:33:52,119 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1f32af3374c6:0 server-side Connection retries=45 2024-11-12T17:33:52,121 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T17:33:52,121 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T17:33:52,122 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T17:33:52,122 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T17:33:52,122 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T17:33:52,126 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T17:33:52,128 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T17:33:52,129 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39493 2024-11-12T17:33:52,131 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39493 connecting to ZooKeeper ensemble=127.0.0.1:61065 2024-11-12T17:33:52,132 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T17:33:52,137 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T17:33:52,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:394930x0, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T17:33:52,148 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:394930x0, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-12T17:33:52,148 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39493-0x1019d8bc6660001 connected 2024-11-12T17:33:52,150 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-12T17:33:52,150 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-12T17:33:52,150 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-12T17:33:52,151 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-12T17:33:52,151 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-12T17:33:52,152 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2024-11-12T17:33:52,152 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-12T17:33:52,153 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-12T17:33:52,153 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-12T17:33:52,155 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 37523 2024-11-12T17:33:52,155 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-12T17:33:52,164 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-12T17:33:52,164 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@13c18cd6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,AVAILABLE} 2024-11-12T17:33:52,165 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-12T17:33:52,165 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@23a380bc{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-12T17:33:52,180 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-12T17:33:52,181 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T17:33:52,181 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T17:33:52,182 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T17:33:52,185 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-12T17:33:52,189 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@6e12a44d{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-12T17:33:52,190 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5804e71e{HTTP/1.1, (http/1.1)}{0.0.0.0:37523} 2024-11-12T17:33:52,190 INFO [Time-limited test {}] server.Server(415): Started @9199ms 2024-11-12T17:33:52,195 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T17:33:52,206 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T17:33:52,208 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2024-11-12T17:33:52,214 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-12T17:33:52,216 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39493 2024-11-12T17:33:52,216 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39493 2024-11-12T17:33:52,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39493 2024-11-12T17:33:52,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39493 2024-11-12T17:33:52,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39493 2024-11-12T17:33:52,243 DEBUG [M:0;1f32af3374c6:39475 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1f32af3374c6:39475 2024-11-12T17:33:52,247 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-12T17:33:52,250 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@4efbcd61{HTTP/1.1, (http/1.1)}{0.0.0.0:37007} 2024-11-12T17:33:52,250 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] server.Server(415): Started @9259ms 2024-11-12T17:33:52,251 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /1/backup-masters/1f32af3374c6,39475,1731432830222 2024-11-12T17:33:52,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-12T17:33:52,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-12T17:33:52,261 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/1f32af3374c6,39475,1731432830222 2024-11-12T17:33:52,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2024-11-12T17:33:52,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:33:52,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:33:52,288 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on existing znode=/1/master 2024-11-12T17:33:52,289 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/1f32af3374c6,39475,1731432830222 from backup master directory 2024-11-12T17:33:52,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/1f32af3374c6,39475,1731432830222 2024-11-12T17:33:52,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-12T17:33:52,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-12T17:33:52,294 WARN [master/1f32af3374c6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T17:33:52,294 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1f32af3374c6,39475,1731432830222 2024-11-12T17:33:52,297 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T17:33:52,299 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T17:33:52,363 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/hbase.id] with ID: 98d01804-71a2-4562-9aba-c6a02e64b7ec 2024-11-12T17:33:52,363 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.tmp/hbase.id 2024-11-12T17:33:52,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741826_1002 (size=42) 2024-11-12T17:33:52,775 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.tmp/hbase.id]:[hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/hbase.id] 2024-11-12T17:33:52,830 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T17:33:52,842 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T17:33:52,872 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 27ms. 2024-11-12T17:33:52,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:33:52,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:33:52,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741827_1003 (size=196) 2024-11-12T17:33:53,314 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:33:53,316 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T17:33:53,325 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-12T17:33:53,327 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-12T17:33:53,339 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.doInit(RegionGroupingProvider.java:156) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.init(AbstractWALProvider.java:126) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:263) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T17:33:53,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741828_1004 (size=1189) 2024-11-12T17:33:53,405 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store 2024-11-12T17:33:53,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741829_1005 (size=34) 2024-11-12T17:33:53,437 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T17:33:53,440 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:33:53,442 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T17:33:53,442 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T17:33:53,442 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T17:33:53,444 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T17:33:53,444 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T17:33:53,444 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T17:33:53,445 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731432833442Disabling compacts and flushes for region at 1731432833442Disabling writes for close at 1731432833444 (+2 ms)Writing region close event to WAL at 1731432833444Closed at 1731432833444 2024-11-12T17:33:53,448 WARN [master/1f32af3374c6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/.initializing 2024-11-12T17:33:53,448 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/WALs/1f32af3374c6,39475,1731432830222 2024-11-12T17:33:53,454 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T17:33:53,461 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1f32af3374c6,39475,1731432830222.regiongroup-0 2024-11-12T17:33:53,483 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1f32af3374c6%2C39475%2C1731432830222.1f32af3374c6%2C39475%2C1731432830222.regiongroup-0, suffix=, logDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/WALs/1f32af3374c6,39475,1731432830222, archiveDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/oldWALs, maxLogs=10 2024-11-12T17:33:53,508 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/WALs/1f32af3374c6,39475,1731432830222/1f32af3374c6%2C39475%2C1731432830222.1f32af3374c6%2C39475%2C1731432830222.regiongroup-0.1731432833488, exclude list is [], retry=0 2024-11-12T17:33:53,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:33:53,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T17:33:53,577 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/WALs/1f32af3374c6,39475,1731432830222/1f32af3374c6%2C39475%2C1731432830222.1f32af3374c6%2C39475%2C1731432830222.regiongroup-0.1731432833488 2024-11-12T17:33:53,578 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:33:53,579 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T17:33:53,579 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:33:53,583 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T17:33:53,585 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T17:33:53,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T17:33:53,675 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T17:33:53,681 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:33:53,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T17:33:53,685 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T17:33:53,696 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T17:33:53,697 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:33:53,698 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:33:53,699 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T17:33:53,702 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T17:33:53,703 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:33:53,704 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:33:53,704 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T17:33:53,708 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T17:33:53,708 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:33:53,709 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:33:53,710 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T17:33:53,714 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T17:33:53,716 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T17:33:53,723 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T17:33:53,724 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T17:33:53,729 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T17:33:53,734 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T17:33:53,741 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T17:33:53,743 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11740248480, jitterRate=0.09339584410190582}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T17:33:53,753 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731432833606Initializing all the Stores at 1731432833609 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731432833609Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432833611 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432833612 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432833612Cleaning up temporary data from old regions at 1731432833724 (+112 ms)Region opened successfully at 1731432833753 (+29 ms) 2024-11-12T17:33:53,755 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T17:33:53,839 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@780ee4c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=1f32af3374c6/172.17.0.2:0 2024-11-12T17:33:53,887 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T17:33:53,904 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T17:33:53,904 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T17:33:53,909 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T17:33:53,911 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-12T17:33:53,918 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-12T17:33:53,919 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T17:33:53,956 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T17:33:53,968 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2024-11-12T17:33:53,971 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/balancer already deleted, retry=false 2024-11-12T17:33:53,974 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T17:33:53,976 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2024-11-12T17:33:53,978 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/normalizer already deleted, retry=false 2024-11-12T17:33:53,980 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T17:33:53,985 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2024-11-12T17:33:53,988 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/split already deleted, retry=false 2024-11-12T17:33:53,990 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2024-11-12T17:33:53,992 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/merge already deleted, retry=false 2024-11-12T17:33:54,018 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T17:33:54,020 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/snapshot-cleanup already deleted, retry=false 2024-11-12T17:33:54,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-12T17:33:54,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-12T17:33:54,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:33:54,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:33:54,030 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1f32af3374c6,39475,1731432830222, sessionid=0x1019d8bc6660000, setting cluster-up flag (Was=false) 2024-11-12T17:33:54,040 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2024-11-12T17:33:54,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:33:54,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:33:54,062 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2024-11-12T17:33:54,064 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1f32af3374c6,39475,1731432830222 2024-11-12T17:33:54,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:33:54,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:33:54,078 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2024-11-12T17:33:54,081 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1f32af3374c6,39475,1731432830222 2024-11-12T17:33:54,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:33:54,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:33:54,096 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2024-11-12T17:33:54,098 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1f32af3374c6,39475,1731432830222 2024-11-12T17:33:54,105 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T17:33:54,129 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(746): ClusterId : 98d01804-71a2-4562-9aba-c6a02e64b7ec 2024-11-12T17:33:54,131 INFO [RS:0;1f32af3374c6:39493 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2024-11-12T17:33:54,133 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T17:33:54,140 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T17:33:54,140 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2024-11-12T17:33:54,144 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2024-11-12T17:33:54,144 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T17:33:54,151 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T17:33:54,152 DEBUG [RS:0;1f32af3374c6:39493 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@702b7390, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=1f32af3374c6/172.17.0.2:0 2024-11-12T17:33:54,168 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1f32af3374c6:39493 2024-11-12T17:33:54,173 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T17:33:54,174 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T17:33:54,174 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T17:33:54,178 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(2659): reportForDuty to master=1f32af3374c6,39475,1731432830222 with port=39493, startcode=1731432832063 2024-11-12T17:33:54,191 DEBUG [RS:0;1f32af3374c6:39493 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T17:33:54,211 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T17:33:54,225 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T17:33:54,234 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42117, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T17:33:54,235 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T17:33:54,242 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39475 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T17:33:54,242 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1f32af3374c6,39475,1731432830222 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T17:33:54,251 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1f32af3374c6:0, corePoolSize=5, maxPoolSize=5 2024-11-12T17:33:54,251 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1f32af3374c6:0, corePoolSize=5, maxPoolSize=5 2024-11-12T17:33:54,251 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1f32af3374c6:0, corePoolSize=5, maxPoolSize=5 2024-11-12T17:33:54,251 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1f32af3374c6:0, corePoolSize=5, maxPoolSize=5 2024-11-12T17:33:54,252 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1f32af3374c6:0, corePoolSize=10, maxPoolSize=10 2024-11-12T17:33:54,252 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1f32af3374c6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T17:33:54,252 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1f32af3374c6:0, corePoolSize=2, maxPoolSize=2 2024-11-12T17:33:54,252 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1f32af3374c6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T17:33:54,259 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T17:33:54,260 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T17:33:54,266 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731432864266 2024-11-12T17:33:54,267 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:33:54,267 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T17:33:54,268 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T17:33:54,269 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T17:33:54,273 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T17:33:54,274 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2024-11-12T17:33:54,274 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T17:33:54,275 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T17:33:54,275 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T17:33:54,278 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-12T17:33:54,279 WARN [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-12T17:33:54,279 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:54,284 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T17:33:54,286 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T17:33:54,286 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T17:33:54,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741831_1007 (size=1321) 2024-11-12T17:33:54,298 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24efbb31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:33:54,298 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:33:54,301 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:33:54,306 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:33:54,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:33:54,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:33:54,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@398f3bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:33:54,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:33:54,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:33:54,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:33:54,322 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:33:54,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74585825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:33:54,325 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-12T17:33:54,328 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T17:33:54,329 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T17:33:54,331 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1f32af3374c6:0:becomeActiveMaster-HFileCleaner.large.0-1731432834330,5,FailOnTimeoutGroup] 2024-11-12T17:33:54,332 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1f32af3374c6:0:becomeActiveMaster-HFileCleaner.small.0-1731432834331,5,FailOnTimeoutGroup] 2024-11-12T17:33:54,332 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:54,332 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T17:33:54,334 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:54,334 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:54,380 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(2659): reportForDuty to master=1f32af3374c6,39475,1731432830222 with port=39493, startcode=1731432832063 2024-11-12T17:33:54,382 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39475 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1f32af3374c6,39493,1731432832063 2024-11-12T17:33:54,385 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39475 {}] master.ServerManager(517): Registering regionserver=1f32af3374c6,39493,1731432832063 2024-11-12T17:33:54,393 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 2024-11-12T17:33:54,393 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:32879 2024-11-12T17:33:54,393 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=41353 2024-11-12T17:33:54,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-12T17:33:54,401 DEBUG [RS:0;1f32af3374c6:39493 {}] zookeeper.ZKUtil(111): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on existing znode=/1/rs/1f32af3374c6,39493,1731432832063 2024-11-12T17:33:54,401 WARN [RS:0;1f32af3374c6:39493 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T17:33:54,401 INFO [RS:0;1f32af3374c6:39493 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-12T17:33:54,402 INFO [RS:0;1f32af3374c6:39493 {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-12T17:33:54,402 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063 2024-11-12T17:33:54,403 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1f32af3374c6,39493,1731432832063] 2024-11-12T17:33:54,428 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T17:33:54,441 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T17:33:54,446 INFO [RS:0;1f32af3374c6:39493 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T17:33:54,447 INFO [RS:0;1f32af3374c6:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:54,447 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T17:33:54,453 INFO [RS:0;1f32af3374c6:39493 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T17:33:54,455 INFO [RS:0;1f32af3374c6:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:54,455 DEBUG [RS:0;1f32af3374c6:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1f32af3374c6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T17:33:54,455 DEBUG [RS:0;1f32af3374c6:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1f32af3374c6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T17:33:54,455 DEBUG [RS:0;1f32af3374c6:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1f32af3374c6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T17:33:54,455 DEBUG [RS:0;1f32af3374c6:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1f32af3374c6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T17:33:54,455 DEBUG [RS:0;1f32af3374c6:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1f32af3374c6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T17:33:54,455 DEBUG [RS:0;1f32af3374c6:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1f32af3374c6:0, corePoolSize=2, maxPoolSize=2 2024-11-12T17:33:54,456 DEBUG [RS:0;1f32af3374c6:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1f32af3374c6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T17:33:54,456 DEBUG [RS:0;1f32af3374c6:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1f32af3374c6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T17:33:54,456 DEBUG [RS:0;1f32af3374c6:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1f32af3374c6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T17:33:54,456 DEBUG [RS:0;1f32af3374c6:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1f32af3374c6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T17:33:54,456 DEBUG [RS:0;1f32af3374c6:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1f32af3374c6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T17:33:54,456 DEBUG [RS:0;1f32af3374c6:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1f32af3374c6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T17:33:54,456 DEBUG [RS:0;1f32af3374c6:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0, corePoolSize=3, maxPoolSize=3 2024-11-12T17:33:54,457 DEBUG [RS:0;1f32af3374c6:39493 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1f32af3374c6:0, corePoolSize=3, maxPoolSize=3 2024-11-12T17:33:54,457 INFO [RS:0;1f32af3374c6:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:54,458 INFO [RS:0;1f32af3374c6:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:54,458 INFO [RS:0;1f32af3374c6:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:54,458 INFO [RS:0;1f32af3374c6:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:54,458 INFO [RS:0;1f32af3374c6:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:54,458 INFO [RS:0;1f32af3374c6:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=1f32af3374c6,39493,1731432832063-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T17:33:54,477 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T17:33:54,479 INFO [RS:0;1f32af3374c6:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=1f32af3374c6,39493,1731432832063-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:54,480 INFO [RS:0;1f32af3374c6:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:54,480 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.Replication(171): 1f32af3374c6,39493,1731432832063 started 2024-11-12T17:33:54,498 INFO [RS:0;1f32af3374c6:39493 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:54,499 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1482): Serving as 1f32af3374c6,39493,1731432832063, RpcServer on 1f32af3374c6/172.17.0.2:39493, sessionid=0x1019d8bc6660001 2024-11-12T17:33:54,500 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T17:33:54,500 DEBUG [RS:0;1f32af3374c6:39493 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1f32af3374c6,39493,1731432832063 2024-11-12T17:33:54,500 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1f32af3374c6,39493,1731432832063' 2024-11-12T17:33:54,500 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2024-11-12T17:33:54,501 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2024-11-12T17:33:54,503 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T17:33:54,503 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2024-11-12T17:33:54,503 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1f32af3374c6,39493,1731432832063' 2024-11-12T17:33:54,503 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T17:33:54,503 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T17:33:54,504 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2024-11-12T17:33:54,504 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2024-11-12T17:33:54,504 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T17:33:54,504 DEBUG [RS:0;1f32af3374c6:39493 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1f32af3374c6,39493,1731432832063 2024-11-12T17:33:54,505 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1f32af3374c6,39493,1731432832063' 2024-11-12T17:33:54,505 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2024-11-12T17:33:54,505 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2024-11-12T17:33:54,506 DEBUG [RS:0;1f32af3374c6:39493 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T17:33:54,506 INFO [RS:0;1f32af3374c6:39493 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T17:33:54,506 INFO [RS:0;1f32af3374c6:39493 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T17:33:54,612 INFO [RS:0;1f32af3374c6:39493 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T17:33:54,613 INFO [RS:0;1f32af3374c6:39493 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1f32af3374c6,39493,1731432832063.regiongroup-0 2024-11-12T17:33:54,617 INFO [RS:0;1f32af3374c6:39493 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0, suffix=, logDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063, archiveDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs, maxLogs=32 2024-11-12T17:33:54,633 DEBUG [RS:0;1f32af3374c6:39493 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432834619, exclude list is [], retry=0 2024-11-12T17:33:54,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:33:54,642 INFO [RS:0;1f32af3374c6:39493 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432834619 2024-11-12T17:33:54,643 DEBUG [RS:0;1f32af3374c6:39493 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:33:54,690 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T17:33:54,691 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 2024-11-12T17:33:54,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741833_1009 (size=32) 2024-11-12T17:33:54,702 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:33:54,704 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T17:33:54,706 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T17:33:54,706 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:33:54,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T17:33:54,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T17:33:54,709 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T17:33:54,709 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:33:54,710 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T17:33:54,710 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T17:33:54,713 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T17:33:54,713 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:33:54,713 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T17:33:54,714 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T17:33:54,716 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T17:33:54,716 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:33:54,717 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T17:33:54,717 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T17:33:54,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740 2024-11-12T17:33:54,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740 2024-11-12T17:33:54,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T17:33:54,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T17:33:54,723 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T17:33:54,725 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T17:33:54,728 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T17:33:54,729 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9460244960, jitterRate=-0.11894603073596954}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T17:33:54,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731432834702Initializing all the Stores at 1731432834703 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731432834703Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731432834704 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432834704Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731432834704Cleaning up temporary data from old regions at 1731432834722 (+18 ms)Region opened successfully at 1731432834732 (+10 ms) 2024-11-12T17:33:54,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T17:33:54,732 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T17:33:54,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T17:33:54,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T17:33:54,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T17:33:54,734 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T17:33:54,734 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731432834732Disabling compacts and flushes for region at 1731432834732Disabling writes for close at 1731432834732Writing region close event to WAL at 1731432834733 (+1 ms)Closed at 1731432834734 (+1 ms) 2024-11-12T17:33:54,739 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T17:33:54,739 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T17:33:54,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T17:33:54,754 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T17:33:54,758 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T17:33:54,912 DEBUG [1f32af3374c6:39475 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-12T17:33:54,928 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:33:54,936 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1f32af3374c6,39493,1731432832063, state=OPENING 2024-11-12T17:33:54,942 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T17:33:54,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:33:54,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:33:54,945 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-12T17:33:54,945 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-12T17:33:54,946 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T17:33:54,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1f32af3374c6,39493,1731432832063}] 2024-11-12T17:33:55,124 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T17:33:55,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47097, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T17:33:55,137 INFO [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T17:33:55,138 INFO [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-12T17:33:55,138 INFO [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-12T17:33:55,138 INFO [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T17:33:55,138 INFO [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T17:33:55,142 INFO [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1f32af3374c6%2C39493%2C1731432832063.meta, suffix=.meta, logDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063, archiveDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs, maxLogs=32 2024-11-12T17:33:55,158 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432835143.meta, exclude list is [], retry=0 2024-11-12T17:33:55,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:33:55,164 INFO [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432835143.meta 2024-11-12T17:33:55,165 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:33:55,165 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T17:33:55,169 INFO [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T17:33:55,170 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T17:33:55,173 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T17:33:55,173 INFO [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T17:33:55,177 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T17:33:55,177 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:33:55,178 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T17:33:55,178 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T17:33:55,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T17:33:55,182 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T17:33:55,183 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:33:55,183 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T17:33:55,184 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T17:33:55,185 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T17:33:55,185 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:33:55,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T17:33:55,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T17:33:55,187 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T17:33:55,187 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:33:55,188 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T17:33:55,188 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T17:33:55,189 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T17:33:55,189 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:33:55,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T17:33:55,190 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T17:33:55,191 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740 2024-11-12T17:33:55,194 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740 2024-11-12T17:33:55,196 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T17:33:55,196 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T17:33:55,197 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T17:33:55,199 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T17:33:55,201 INFO [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9476151520, jitterRate=-0.11746461689472198}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T17:33:55,201 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T17:33:55,203 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731432835178Writing region info on filesystem at 1731432835179 (+1 ms)Initializing all the Stores at 1731432835180 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731432835181 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731432835181Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432835181Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731432835181Cleaning up temporary data from old regions at 1731432835196 (+15 ms)Running coprocessor post-open hooks at 1731432835201 (+5 ms)Region opened successfully at 1731432835203 (+2 ms) 2024-11-12T17:33:55,212 INFO [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731432835116 2024-11-12T17:33:55,225 DEBUG [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T17:33:55,226 INFO [RS_OPEN_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T17:33:55,228 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:33:55,231 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1f32af3374c6,39493,1731432832063, state=OPEN 2024-11-12T17:33:55,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-12T17:33:55,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-12T17:33:55,234 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-12T17:33:55,234 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-12T17:33:55,234 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1f32af3374c6,39493,1731432832063 2024-11-12T17:33:55,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T17:33:55,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1f32af3374c6,39493,1731432832063 in 286 msec 2024-11-12T17:33:55,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T17:33:55,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 496 msec 2024-11-12T17:33:55,250 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T17:33:55,250 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T17:33:55,273 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:33:55,274 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:33:55,300 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:33:55,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54469, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:33:55,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1800 sec 2024-11-12T17:33:55,325 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731432835325, completionTime=-1 2024-11-12T17:33:55,328 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-12T17:33:55,328 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T17:33:55,357 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-12T17:33:55,357 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731432895357 2024-11-12T17:33:55,357 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731432955357 2024-11-12T17:33:55,357 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-12T17:33:55,359 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1f32af3374c6,39475,1731432830222-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:55,359 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1f32af3374c6,39475,1731432830222-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:55,360 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1f32af3374c6,39475,1731432830222-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:55,361 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1f32af3374c6:39475, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:55,362 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:55,362 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:55,369 DEBUG [master/1f32af3374c6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T17:33:55,404 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.109sec 2024-11-12T17:33:55,406 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T17:33:55,408 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T17:33:55,409 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T17:33:55,410 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T17:33:55,410 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T17:33:55,411 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1f32af3374c6,39475,1731432830222-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T17:33:55,412 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1f32af3374c6,39475,1731432830222-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T17:33:55,423 DEBUG [master/1f32af3374c6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T17:33:55,425 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T17:33:55,425 INFO [master/1f32af3374c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1f32af3374c6,39475,1731432830222-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T17:33:55,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b200574, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:33:55,434 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T17:33:55,434 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T17:33:55,439 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:33:55,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:33:55,449 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:33:55,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:33:55,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:33:55,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c7b4754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:33:55,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:33:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:33:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:33:55,464 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:33:55,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ea5228, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:33:55,467 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:33:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:33:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:33:55,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:33:55,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1f32af3374c6,39475,1731432830222 2024-11-12T17:33:55,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2305): Starting mini mapreduce cluster... 2024-11-12T17:33:55,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/test.cache.data in system properties and HBase conf 2024-11-12T17:33:55,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T17:33:55,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir in system properties and HBase conf 2024-11-12T17:33:55,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T17:33:55,502 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T17:33:55,502 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T17:33:55,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T17:33:55,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T17:33:55,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T17:33:55,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T17:33:55,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T17:33:55,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T17:33:55,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T17:33:55,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T17:33:55,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T17:33:55,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/nfs.dump.dir in system properties and HBase conf 2024-11-12T17:33:55,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/java.io.tmpdir in system properties and HBase conf 2024-11-12T17:33:55,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T17:33:55,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T17:33:55,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T17:33:55,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741835_1011 (size=592039) 2024-11-12T17:33:55,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741836_1012 (size=1663647) 2024-11-12T17:33:57,671 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T17:33:57,793 WARN [Thread-251 {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T17:33:58,008 INFO [Thread-251 {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T17:33:58,009 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-12T17:33:58,010 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T17:33:58,011 INFO [Thread-251 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T17:33:58,011 INFO [Thread-251 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T17:33:58,011 INFO [Thread-251 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T17:33:58,012 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a6206d8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,AVAILABLE} 2024-11-12T17:33:58,013 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@338b53fb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-12T17:33:58,024 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T17:33:58,024 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T17:33:58,024 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T17:33:58,026 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T17:33:58,034 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@426811c2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,AVAILABLE} 2024-11-12T17:33:58,035 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ccdffea{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-12T17:33:58,183 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2024-11-12T17:33:58,184 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2024-11-12T17:33:58,184 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-12T17:33:58,186 INFO [Thread-251 {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-12T17:33:58,230 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T17:33:58,339 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T17:33:58,596 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T17:33:58,625 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@56fe9299{jobhistory,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/java.io.tmpdir/jetty-localhost-43025-hadoop-yarn-common-3_4_1_jar-_-any-16583123211490300606/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-12T17:33:58,625 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6428151{cluster,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/java.io.tmpdir/jetty-localhost-36319-hadoop-yarn-common-3_4_1_jar-_-any-5979919745832433922/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-12T17:33:58,626 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@440bd171{HTTP/1.1, (http/1.1)}{localhost:36319} 2024-11-12T17:33:58,626 INFO [Time-limited test {}] server.Server(415): Started @15635ms 2024-11-12T17:33:58,628 INFO [Thread-251 {}] server.AbstractConnector(333): Started ServerConnector@1478f1aa{HTTP/1.1, (http/1.1)}{localhost:43025} 2024-11-12T17:33:58,628 INFO [Thread-251 {}] server.Server(415): Started @15637ms 2024-11-12T17:33:58,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741837_1013 (size=5) 2024-11-12T17:34:00,038 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-12T17:34:00,043 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T17:34:00,078 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-12T17:34:00,079 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T17:34:00,085 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T17:34:00,085 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T17:34:00,085 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T17:34:00,088 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T17:34:00,089 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b88ca19{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,AVAILABLE} 2024-11-12T17:34:00,089 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@444b3a56{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-12T17:34:00,166 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-12T17:34:00,166 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-12T17:34:00,166 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-12T17:34:00,166 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-12T17:34:00,175 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T17:34:00,183 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T17:34:00,286 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T17:34:00,297 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@380ba3dd{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/java.io.tmpdir/jetty-localhost-39799-hadoop-yarn-common-3_4_1_jar-_-any-8883547595816636619/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-12T17:34:00,298 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@279f726d{HTTP/1.1, (http/1.1)}{localhost:39799} 2024-11-12T17:34:00,298 INFO [Time-limited test {}] server.Server(415): Started @17307ms 2024-11-12T17:34:00,434 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-12T17:34:00,437 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T17:34:00,452 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-12T17:34:00,452 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T17:34:00,454 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T17:34:00,454 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T17:34:00,454 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T17:34:00,455 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T17:34:00,456 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2070a37b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,AVAILABLE} 2024-11-12T17:34:00,457 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7eed520b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-12T17:34:00,523 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-12T17:34:00,523 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-12T17:34:00,524 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-12T17:34:00,524 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-12T17:34:00,535 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T17:34:00,539 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T17:34:00,642 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T17:34:00,646 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@29a5b7e8{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/java.io.tmpdir/jetty-localhost-44069-hadoop-yarn-common-3_4_1_jar-_-any-12729836547781317816/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-12T17:34:00,647 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3b1e9d33{HTTP/1.1, (http/1.1)}{localhost:44069} 2024-11-12T17:34:00,647 INFO [Time-limited test {}] server.Server(415): Started @17656ms 2024-11-12T17:34:00,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2341): Mini mapreduce cluster started 2024-11-12T17:34:00,686 INFO [Time-limited test {}] backup.TestBackupBase(336): ROOTDIR hdfs://localhost:32879/backupUT 2024-11-12T17:34:00,690 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T17:34:00,691 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-12T17:34:00,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1f32af3374c6,39475,1731432830222 2024-11-12T17:34:00,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3d546a81 2024-11-12T17:34:00,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T17:34:00,708 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T17:34:00,711 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns1'} 2024-11-12T17:34:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns1 2024-11-12T17:34:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T17:34:00,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns1 in 21 msec 2024-11-12T17:34:00,767 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T17:34:00,770 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T17:34:00,770 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T17:34:00,773 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T17:34:00,773 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2024-11-12T17:34:00,773 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T17:34:00,773 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T17:34:00,774 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T17:34:00,774 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T17:34:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T17:34:00,843 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns1 completed 2024-11-12T17:34:00,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns2'} 2024-11-12T17:34:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns2 2024-11-12T17:34:00,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-12T17:34:00,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns2 in 13 msec 2024-11-12T17:34:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-12T17:34:00,962 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns2 completed 2024-11-12T17:34:00,963 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns3'} 2024-11-12T17:34:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=6, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns3 2024-11-12T17:34:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-12T17:34:00,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns3 in 13 msec 2024-11-12T17:34:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-12T17:34:01,073 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns3 completed 2024-11-12T17:34:01,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns4'} 2024-11-12T17:34:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns4 2024-11-12T17:34:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T17:34:01,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns4 in 11 msec 2024-11-12T17:34:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T17:34:01,183 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns4 completed 2024-11-12T17:34:01,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test-1731432840686', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:34:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test-1731432840686 2024-11-12T17:34:01,203 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test-1731432840686 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T17:34:01,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test-1731432840686" procId is: 8 2024-11-12T17:34:01,205 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-12T17:34:01,207 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test-1731432840686 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T17:34:01,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741838_1014 (size=367) 2024-11-12T17:34:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-12T17:34:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-12T17:34:01,621 INFO [RegionOpenAndInit-test-1731432840686-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f14197f5e90bb7bd826d4a3ae95e1d7e, NAME => 'test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1731432840686', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 2024-11-12T17:34:01,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741839_1015 (size=53) 2024-11-12T17:34:01,630 DEBUG [RegionOpenAndInit-test-1731432840686-pool-0 {}] regionserver.HRegion(898): Instantiated test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:34:01,630 DEBUG [RegionOpenAndInit-test-1731432840686-pool-0 {}] regionserver.HRegion(1722): Closing f14197f5e90bb7bd826d4a3ae95e1d7e, disabling compactions & flushes 2024-11-12T17:34:01,630 INFO [RegionOpenAndInit-test-1731432840686-pool-0 {}] regionserver.HRegion(1755): Closing region test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:01,630 DEBUG [RegionOpenAndInit-test-1731432840686-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:01,630 DEBUG [RegionOpenAndInit-test-1731432840686-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. after waiting 0 ms 2024-11-12T17:34:01,631 DEBUG [RegionOpenAndInit-test-1731432840686-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:01,631 INFO [RegionOpenAndInit-test-1731432840686-pool-0 {}] regionserver.HRegion(1973): Closed test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:01,631 DEBUG [RegionOpenAndInit-test-1731432840686-pool-0 {}] regionserver.HRegion(1676): Region close journal for f14197f5e90bb7bd826d4a3ae95e1d7e: Waiting for close lock at 1731432841630Disabling compacts and flushes for region at 1731432841630Disabling writes for close at 1731432841630Writing region close event to WAL at 1731432841631 (+1 ms)Closed at 1731432841631 2024-11-12T17:34:01,633 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test-1731432840686 execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T17:34:01,639 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731432841633"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731432841633"}]},"ts":"1731432841633"} 2024-11-12T17:34:01,643 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T17:34:01,645 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test-1731432840686 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T17:34:01,648 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1731432840686","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432841645"}]},"ts":"1731432841645"} 2024-11-12T17:34:01,652 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1731432840686, state=ENABLING in hbase:meta 2024-11-12T17:34:01,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, ASSIGN}] 2024-11-12T17:34:01,656 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, ASSIGN 2024-11-12T17:34:01,657 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, ASSIGN; state=OFFLINE, location=1f32af3374c6,39493,1731432832063; forceNewPlan=false, retain=false 2024-11-12T17:34:01,809 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=f14197f5e90bb7bd826d4a3ae95e1d7e, regionState=OPENING, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:01,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, ASSIGN because future has completed 2024-11-12T17:34:01,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e, server=1f32af3374c6,39493,1731432832063}] 2024-11-12T17:34:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-12T17:34:01,999 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(132): Open test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:01,999 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T17:34:02,000 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] monitor.StreamSlowMonitor(122): New stream slow monitor 1f32af3374c6,39493,1731432832063.regiongroup-1 2024-11-12T17:34:02,002 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1, suffix=, logDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063, archiveDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs, maxLogs=32 2024-11-12T17:34:02,019 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432842004, exclude list is [], retry=0 2024-11-12T17:34:02,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:34:02,025 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432842004 2024-11-12T17:34:02,026 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:34:02,026 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7752): Opening region: {ENCODED => f14197f5e90bb7bd826d4a3ae95e1d7e, NAME => 'test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e.', STARTKEY => '', ENDKEY => ''} 2024-11-12T17:34:02,026 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T17:34:02,027 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1731432840686 f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:02,027 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(898): Instantiated test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:34:02,027 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7794): checking encryption for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:02,027 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7797): checking classloading for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:02,029 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:02,032 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f14197f5e90bb7bd826d4a3ae95e1d7e columnFamilyName f 2024-11-12T17:34:02,032 DEBUG [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:02,033 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.HStore(327): Store=f14197f5e90bb7bd826d4a3ae95e1d7e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:34:02,033 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1038): replaying wal for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:02,035 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:02,035 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:02,036 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1048): stopping wal replay for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:02,036 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1060): Cleaning up temporary data for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:02,039 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1093): writing seq id for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:02,042 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T17:34:02,043 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1114): Opened f14197f5e90bb7bd826d4a3ae95e1d7e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11177115040, jitterRate=0.04094995558261871}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T17:34:02,043 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:02,044 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1006): Region open journal for f14197f5e90bb7bd826d4a3ae95e1d7e: Running coprocessor pre-open hook at 1731432842027Writing region info on filesystem at 1731432842027Initializing all the Stores at 1731432842029 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432842029Cleaning up temporary data from old regions at 1731432842036 (+7 ms)Running coprocessor post-open hooks at 1731432842043 (+7 ms)Region opened successfully at 1731432842044 (+1 ms) 2024-11-12T17:34:02,046 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e., pid=10, masterSystemTime=1731432841968 2024-11-12T17:34:02,049 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:02,049 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(153): Opened test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:02,050 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=f14197f5e90bb7bd826d4a3ae95e1d7e, regionState=OPEN, openSeqNum=2, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:02,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e, server=1f32af3374c6,39493,1731432832063 because future has completed 2024-11-12T17:34:02,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-12T17:34:02,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e, server=1f32af3374c6,39493,1731432832063 in 241 msec 2024-11-12T17:34:02,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-12T17:34:02,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, ASSIGN in 405 msec 2024-11-12T17:34:02,063 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test-1731432840686 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T17:34:02,064 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1731432840686","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432842063"}]},"ts":"1731432842063"} 2024-11-12T17:34:02,066 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1731432840686, state=ENABLED in hbase:meta 2024-11-12T17:34:02,068 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test-1731432840686 execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T17:34:02,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; CreateTableProcedure table=test-1731432840686 in 873 msec 2024-11-12T17:34:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-12T17:34:02,342 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test-1731432840686 completed 2024-11-12T17:34:02,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625aeb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:02,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:02,344 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:02,345 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f59870, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:02,349 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eddd0a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:02,353 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:02,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:02,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:02,357 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1731432840686', row='row0', locateType=CURRENT is [region=test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e., hostname=1f32af3374c6,39493,1731432832063, seqNum=2] 2024-11-12T17:34:02,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39493 {}] regionserver.HRegion(8528): writing data to region test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. with WAL disabled. Data may be lost in the event of a crash. 2024-11-12T17:34:02,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns2:test-17314328406861', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:34:02,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:test-17314328406861 2024-11-12T17:34:02,531 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17314328406861 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T17:34:02,531 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:02,531 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns2" qualifier: "test-17314328406861" procId is: 11 2024-11-12T17:34:02,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T17:34:02,532 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:test-17314328406861 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T17:34:02,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741841_1017 (size=364) 2024-11-12T17:34:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T17:34:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T17:34:02,945 INFO [RegionOpenAndInit-ns2:test-17314328406861-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6dd38527fc6c5b38bfed421a5b5be067, NAME => 'ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17314328406861', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 2024-11-12T17:34:02,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741842_1018 (size=50) 2024-11-12T17:34:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T17:34:03,356 DEBUG [RegionOpenAndInit-ns2:test-17314328406861-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:34:03,357 DEBUG [RegionOpenAndInit-ns2:test-17314328406861-pool-0 {}] regionserver.HRegion(1722): Closing 6dd38527fc6c5b38bfed421a5b5be067, disabling compactions & flushes 2024-11-12T17:34:03,357 INFO [RegionOpenAndInit-ns2:test-17314328406861-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:34:03,357 DEBUG [RegionOpenAndInit-ns2:test-17314328406861-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:34:03,357 DEBUG [RegionOpenAndInit-ns2:test-17314328406861-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. after waiting 0 ms 2024-11-12T17:34:03,357 DEBUG [RegionOpenAndInit-ns2:test-17314328406861-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:34:03,357 INFO [RegionOpenAndInit-ns2:test-17314328406861-pool-0 {}] regionserver.HRegion(1973): Closed ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:34:03,357 DEBUG [RegionOpenAndInit-ns2:test-17314328406861-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6dd38527fc6c5b38bfed421a5b5be067: Waiting for close lock at 1731432843357Disabling compacts and flushes for region at 1731432843357Disabling writes for close at 1731432843357Writing region close event to WAL at 1731432843357Closed at 1731432843357 2024-11-12T17:34:03,359 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:test-17314328406861 execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T17:34:03,359 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1731432843359"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731432843359"}]},"ts":"1731432843359"} 2024-11-12T17:34:03,366 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T17:34:03,368 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:test-17314328406861 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T17:34:03,369 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17314328406861","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432843368"}]},"ts":"1731432843368"} 2024-11-12T17:34:03,371 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17314328406861, state=ENABLING in hbase:meta 2024-11-12T17:34:03,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17314328406861, region=6dd38527fc6c5b38bfed421a5b5be067, ASSIGN}] 2024-11-12T17:34:03,374 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17314328406861, region=6dd38527fc6c5b38bfed421a5b5be067, ASSIGN 2024-11-12T17:34:03,375 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:test-17314328406861, region=6dd38527fc6c5b38bfed421a5b5be067, ASSIGN; state=OFFLINE, location=1f32af3374c6,39493,1731432832063; forceNewPlan=false, retain=false 2024-11-12T17:34:03,526 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=6dd38527fc6c5b38bfed421a5b5be067, regionState=OPENING, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:03,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:test-17314328406861, region=6dd38527fc6c5b38bfed421a5b5be067, ASSIGN because future has completed 2024-11-12T17:34:03,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6dd38527fc6c5b38bfed421a5b5be067, server=1f32af3374c6,39493,1731432832063}] 2024-11-12T17:34:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T17:34:03,689 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:34:03,689 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 6dd38527fc6c5b38bfed421a5b5be067, NAME => 'ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067.', STARTKEY => '', ENDKEY => ''} 2024-11-12T17:34:03,690 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T17:34:03,690 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17314328406861 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:03,690 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:34:03,690 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:03,690 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:03,693 INFO [StoreOpener-6dd38527fc6c5b38bfed421a5b5be067-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:03,695 INFO [StoreOpener-6dd38527fc6c5b38bfed421a5b5be067-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6dd38527fc6c5b38bfed421a5b5be067 columnFamilyName f 2024-11-12T17:34:03,695 DEBUG [StoreOpener-6dd38527fc6c5b38bfed421a5b5be067-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:03,696 INFO [StoreOpener-6dd38527fc6c5b38bfed421a5b5be067-1 {}] regionserver.HStore(327): Store=6dd38527fc6c5b38bfed421a5b5be067/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:34:03,696 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:03,697 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:03,698 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:03,699 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:03,699 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:03,701 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:03,704 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T17:34:03,705 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 6dd38527fc6c5b38bfed421a5b5be067; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9495064000, jitterRate=-0.11570325493812561}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T17:34:03,705 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:03,707 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 6dd38527fc6c5b38bfed421a5b5be067: Running coprocessor pre-open hook at 1731432843690Writing region info on filesystem at 1731432843690Initializing all the Stores at 1731432843692 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432843692Cleaning up temporary data from old regions at 1731432843699 (+7 ms)Running coprocessor post-open hooks at 1731432843705 (+6 ms)Region opened successfully at 1731432843706 (+1 ms) 2024-11-12T17:34:03,708 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067., pid=13, masterSystemTime=1731432843684 2024-11-12T17:34:03,711 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:34:03,711 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:34:03,712 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=6dd38527fc6c5b38bfed421a5b5be067, regionState=OPEN, openSeqNum=2, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:03,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6dd38527fc6c5b38bfed421a5b5be067, server=1f32af3374c6,39493,1731432832063 because future has completed 2024-11-12T17:34:03,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-11-12T17:34:03,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 6dd38527fc6c5b38bfed421a5b5be067, server=1f32af3374c6,39493,1731432832063 in 186 msec 2024-11-12T17:34:03,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-12T17:34:03,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:test-17314328406861, region=6dd38527fc6c5b38bfed421a5b5be067, ASSIGN in 348 msec 2024-11-12T17:34:03,724 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:test-17314328406861 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T17:34:03,724 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17314328406861","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432843724"}]},"ts":"1731432843724"} 2024-11-12T17:34:03,727 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17314328406861, state=ENABLED in hbase:meta 2024-11-12T17:34:03,728 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17314328406861 execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T17:34:03,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:test-17314328406861 in 1.2010 sec 2024-11-12T17:34:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T17:34:04,692 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:test-17314328406861 completed 2024-11-12T17:34:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17314328406861', row='row0', locateType=CURRENT is [region=ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067., hostname=1f32af3374c6,39493,1731432832063, seqNum=2] 2024-11-12T17:34:04,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39493 {}] regionserver.HRegion(8528): writing data to region ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. with WAL disabled. Data may be lost in the event of a crash. 2024-11-12T17:34:04,873 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns3:test-17314328406862', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:34:04,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns3:test-17314328406862 2024-11-12T17:34:04,877 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17314328406862 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T17:34:04,878 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:04,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns3" qualifier: "test-17314328406862" procId is: 14 2024-11-12T17:34:04,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-12T17:34:04,879 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns3:test-17314328406862 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T17:34:04,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741843_1019 (size=390) 2024-11-12T17:34:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-12T17:34:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-12T17:34:05,290 INFO [RegionOpenAndInit-ns3:test-17314328406862-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ad7fb5b314f3c7778103a7cc5221da50, NAME => 'ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17314328406862', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 2024-11-12T17:34:05,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741844_1020 (size=50) 2024-11-12T17:34:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-12T17:34:05,700 DEBUG [RegionOpenAndInit-ns3:test-17314328406862-pool-0 {}] regionserver.HRegion(898): Instantiated ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:34:05,700 DEBUG [RegionOpenAndInit-ns3:test-17314328406862-pool-0 {}] regionserver.HRegion(1722): Closing ad7fb5b314f3c7778103a7cc5221da50, disabling compactions & flushes 2024-11-12T17:34:05,700 INFO [RegionOpenAndInit-ns3:test-17314328406862-pool-0 {}] regionserver.HRegion(1755): Closing region ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. 2024-11-12T17:34:05,700 DEBUG [RegionOpenAndInit-ns3:test-17314328406862-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. 2024-11-12T17:34:05,700 DEBUG [RegionOpenAndInit-ns3:test-17314328406862-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. after waiting 0 ms 2024-11-12T17:34:05,700 DEBUG [RegionOpenAndInit-ns3:test-17314328406862-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. 2024-11-12T17:34:05,700 INFO [RegionOpenAndInit-ns3:test-17314328406862-pool-0 {}] regionserver.HRegion(1973): Closed ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. 2024-11-12T17:34:05,700 DEBUG [RegionOpenAndInit-ns3:test-17314328406862-pool-0 {}] regionserver.HRegion(1676): Region close journal for ad7fb5b314f3c7778103a7cc5221da50: Waiting for close lock at 1731432845700Disabling compacts and flushes for region at 1731432845700Disabling writes for close at 1731432845700Writing region close event to WAL at 1731432845700Closed at 1731432845700 2024-11-12T17:34:05,702 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns3:test-17314328406862 execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T17:34:05,703 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1731432845702"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731432845702"}]},"ts":"1731432845702"} 2024-11-12T17:34:05,706 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T17:34:05,707 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns3:test-17314328406862 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T17:34:05,707 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17314328406862","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432845707"}]},"ts":"1731432845707"} 2024-11-12T17:34:05,709 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17314328406862, state=ENABLING in hbase:meta 2024-11-12T17:34:05,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17314328406862, region=ad7fb5b314f3c7778103a7cc5221da50, ASSIGN}] 2024-11-12T17:34:05,711 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17314328406862, region=ad7fb5b314f3c7778103a7cc5221da50, ASSIGN 2024-11-12T17:34:05,713 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns3:test-17314328406862, region=ad7fb5b314f3c7778103a7cc5221da50, ASSIGN; state=OFFLINE, location=1f32af3374c6,39493,1731432832063; forceNewPlan=false, retain=false 2024-11-12T17:34:05,863 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=ad7fb5b314f3c7778103a7cc5221da50, regionState=OPENING, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:05,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns3:test-17314328406862, region=ad7fb5b314f3c7778103a7cc5221da50, ASSIGN because future has completed 2024-11-12T17:34:05,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad7fb5b314f3c7778103a7cc5221da50, server=1f32af3374c6,39493,1731432832063}] 2024-11-12T17:34:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-12T17:34:06,025 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. 2024-11-12T17:34:06,026 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => ad7fb5b314f3c7778103a7cc5221da50, NAME => 'ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50.', STARTKEY => '', ENDKEY => ''} 2024-11-12T17:34:06,026 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T17:34:06,026 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17314328406862 ad7fb5b314f3c7778103a7cc5221da50 2024-11-12T17:34:06,026 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:34:06,026 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for ad7fb5b314f3c7778103a7cc5221da50 2024-11-12T17:34:06,027 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for ad7fb5b314f3c7778103a7cc5221da50 2024-11-12T17:34:06,029 INFO [StoreOpener-ad7fb5b314f3c7778103a7cc5221da50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region ad7fb5b314f3c7778103a7cc5221da50 2024-11-12T17:34:06,033 INFO [StoreOpener-ad7fb5b314f3c7778103a7cc5221da50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad7fb5b314f3c7778103a7cc5221da50 columnFamilyName f 2024-11-12T17:34:06,033 DEBUG [StoreOpener-ad7fb5b314f3c7778103a7cc5221da50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:06,039 INFO [StoreOpener-ad7fb5b314f3c7778103a7cc5221da50-1 {}] regionserver.HStore(327): Store=ad7fb5b314f3c7778103a7cc5221da50/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:34:06,039 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for ad7fb5b314f3c7778103a7cc5221da50 2024-11-12T17:34:06,040 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns3/test-17314328406862/ad7fb5b314f3c7778103a7cc5221da50 2024-11-12T17:34:06,041 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns3/test-17314328406862/ad7fb5b314f3c7778103a7cc5221da50 2024-11-12T17:34:06,042 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for ad7fb5b314f3c7778103a7cc5221da50 2024-11-12T17:34:06,042 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for ad7fb5b314f3c7778103a7cc5221da50 2024-11-12T17:34:06,045 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for ad7fb5b314f3c7778103a7cc5221da50 2024-11-12T17:34:06,048 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns3/test-17314328406862/ad7fb5b314f3c7778103a7cc5221da50/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T17:34:06,049 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened ad7fb5b314f3c7778103a7cc5221da50; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9559509760, jitterRate=-0.10970127582550049}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T17:34:06,049 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ad7fb5b314f3c7778103a7cc5221da50 2024-11-12T17:34:06,050 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for ad7fb5b314f3c7778103a7cc5221da50: Running coprocessor pre-open hook at 1731432846027Writing region info on filesystem at 1731432846027Initializing all the Stores at 1731432846028 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432846029 (+1 ms)Cleaning up temporary data from old regions at 1731432846042 (+13 ms)Running coprocessor post-open hooks at 1731432846049 (+7 ms)Region opened successfully at 1731432846050 (+1 ms) 2024-11-12T17:34:06,051 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50., pid=16, masterSystemTime=1731432846020 2024-11-12T17:34:06,054 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. 2024-11-12T17:34:06,054 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. 2024-11-12T17:34:06,055 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=ad7fb5b314f3c7778103a7cc5221da50, regionState=OPEN, openSeqNum=2, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:06,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad7fb5b314f3c7778103a7cc5221da50, server=1f32af3374c6,39493,1731432832063 because future has completed 2024-11-12T17:34:06,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-12T17:34:06,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure ad7fb5b314f3c7778103a7cc5221da50, server=1f32af3374c6,39493,1731432832063 in 193 msec 2024-11-12T17:34:06,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-12T17:34:06,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns3:test-17314328406862, region=ad7fb5b314f3c7778103a7cc5221da50, ASSIGN in 352 msec 2024-11-12T17:34:06,067 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns3:test-17314328406862 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T17:34:06,067 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17314328406862","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432846067"}]},"ts":"1731432846067"} 2024-11-12T17:34:06,069 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17314328406862, state=ENABLED in hbase:meta 2024-11-12T17:34:06,070 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17314328406862 execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T17:34:06,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns3:test-17314328406862 in 1.1970 sec 2024-11-12T17:34:06,770 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17314328406862' 2024-11-12T17:34:06,770 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17314328406861' 2024-11-12T17:34:06,771 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1731432840686' 2024-11-12T17:34:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-12T17:34:07,022 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns3:test-17314328406862 completed 2024-11-12T17:34:07,023 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns3:test-17314328406862 get assigned. Timeout = 60000ms 2024-11-12T17:34:07,024 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T17:34:07,030 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns3:test-17314328406862 assigned to meta. Checking AM states. 2024-11-12T17:34:07,031 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T17:34:07,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns3:test-17314328406862 assigned. 2024-11-12T17:34:07,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns4:test-17314328406863', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:34:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns4:test-17314328406863 2024-11-12T17:34:07,037 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17314328406863 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T17:34:07,037 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:07,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns4" qualifier: "test-17314328406863" procId is: 17 2024-11-12T17:34:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T17:34:07,038 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns4:test-17314328406863 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T17:34:07,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741845_1021 (size=390) 2024-11-12T17:34:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T17:34:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T17:34:07,449 INFO [RegionOpenAndInit-ns4:test-17314328406863-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a2efbf6b09e33aa73d99cb054d7d7f3e, NAME => 'ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17314328406863', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 2024-11-12T17:34:07,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741846_1022 (size=50) 2024-11-12T17:34:07,530 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-12T17:34:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T17:34:07,860 DEBUG [RegionOpenAndInit-ns4:test-17314328406863-pool-0 {}] regionserver.HRegion(898): Instantiated ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:34:07,860 DEBUG [RegionOpenAndInit-ns4:test-17314328406863-pool-0 {}] regionserver.HRegion(1722): Closing a2efbf6b09e33aa73d99cb054d7d7f3e, disabling compactions & flushes 2024-11-12T17:34:07,860 INFO [RegionOpenAndInit-ns4:test-17314328406863-pool-0 {}] regionserver.HRegion(1755): Closing region ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. 2024-11-12T17:34:07,860 DEBUG [RegionOpenAndInit-ns4:test-17314328406863-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. 2024-11-12T17:34:07,860 DEBUG [RegionOpenAndInit-ns4:test-17314328406863-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. after waiting 0 ms 2024-11-12T17:34:07,860 DEBUG [RegionOpenAndInit-ns4:test-17314328406863-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. 2024-11-12T17:34:07,860 INFO [RegionOpenAndInit-ns4:test-17314328406863-pool-0 {}] regionserver.HRegion(1973): Closed ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. 2024-11-12T17:34:07,860 DEBUG [RegionOpenAndInit-ns4:test-17314328406863-pool-0 {}] regionserver.HRegion(1676): Region close journal for a2efbf6b09e33aa73d99cb054d7d7f3e: Waiting for close lock at 1731432847860Disabling compacts and flushes for region at 1731432847860Disabling writes for close at 1731432847860Writing region close event to WAL at 1731432847860Closed at 1731432847860 2024-11-12T17:34:07,862 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns4:test-17314328406863 execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T17:34:07,863 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1731432847862"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731432847862"}]},"ts":"1731432847862"} 2024-11-12T17:34:07,866 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T17:34:07,867 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns4:test-17314328406863 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T17:34:07,868 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17314328406863","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432847867"}]},"ts":"1731432847867"} 2024-11-12T17:34:07,870 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17314328406863, state=ENABLING in hbase:meta 2024-11-12T17:34:07,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17314328406863, region=a2efbf6b09e33aa73d99cb054d7d7f3e, ASSIGN}] 2024-11-12T17:34:07,872 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17314328406863, region=a2efbf6b09e33aa73d99cb054d7d7f3e, ASSIGN 2024-11-12T17:34:07,874 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns4:test-17314328406863, region=a2efbf6b09e33aa73d99cb054d7d7f3e, ASSIGN; state=OFFLINE, location=1f32af3374c6,39493,1731432832063; forceNewPlan=false, retain=false 2024-11-12T17:34:08,025 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=a2efbf6b09e33aa73d99cb054d7d7f3e, regionState=OPENING, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:08,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns4:test-17314328406863, region=a2efbf6b09e33aa73d99cb054d7d7f3e, ASSIGN because future has completed 2024-11-12T17:34:08,029 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2efbf6b09e33aa73d99cb054d7d7f3e, server=1f32af3374c6,39493,1731432832063}] 2024-11-12T17:34:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T17:34:08,187 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. 2024-11-12T17:34:08,187 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => a2efbf6b09e33aa73d99cb054d7d7f3e, NAME => 'ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e.', STARTKEY => '', ENDKEY => ''} 2024-11-12T17:34:08,188 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T17:34:08,188 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17314328406863 a2efbf6b09e33aa73d99cb054d7d7f3e 2024-11-12T17:34:08,188 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:34:08,188 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for a2efbf6b09e33aa73d99cb054d7d7f3e 2024-11-12T17:34:08,188 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for a2efbf6b09e33aa73d99cb054d7d7f3e 2024-11-12T17:34:08,190 INFO [StoreOpener-a2efbf6b09e33aa73d99cb054d7d7f3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a2efbf6b09e33aa73d99cb054d7d7f3e 2024-11-12T17:34:08,193 INFO [StoreOpener-a2efbf6b09e33aa73d99cb054d7d7f3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2efbf6b09e33aa73d99cb054d7d7f3e columnFamilyName f 2024-11-12T17:34:08,193 DEBUG [StoreOpener-a2efbf6b09e33aa73d99cb054d7d7f3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:08,194 INFO [StoreOpener-a2efbf6b09e33aa73d99cb054d7d7f3e-1 {}] regionserver.HStore(327): Store=a2efbf6b09e33aa73d99cb054d7d7f3e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:34:08,194 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for a2efbf6b09e33aa73d99cb054d7d7f3e 2024-11-12T17:34:08,195 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns4/test-17314328406863/a2efbf6b09e33aa73d99cb054d7d7f3e 2024-11-12T17:34:08,196 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns4/test-17314328406863/a2efbf6b09e33aa73d99cb054d7d7f3e 2024-11-12T17:34:08,196 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for a2efbf6b09e33aa73d99cb054d7d7f3e 2024-11-12T17:34:08,196 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for a2efbf6b09e33aa73d99cb054d7d7f3e 2024-11-12T17:34:08,199 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for a2efbf6b09e33aa73d99cb054d7d7f3e 2024-11-12T17:34:08,202 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns4/test-17314328406863/a2efbf6b09e33aa73d99cb054d7d7f3e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T17:34:08,202 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened a2efbf6b09e33aa73d99cb054d7d7f3e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11356028000, jitterRate=0.057612523436546326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T17:34:08,202 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a2efbf6b09e33aa73d99cb054d7d7f3e 2024-11-12T17:34:08,203 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for a2efbf6b09e33aa73d99cb054d7d7f3e: Running coprocessor pre-open hook at 1731432848188Writing region info on filesystem at 1731432848188Initializing all the Stores at 1731432848190 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432848190Cleaning up temporary data from old regions at 1731432848196 (+6 ms)Running coprocessor post-open hooks at 1731432848203 (+7 ms)Region opened successfully at 1731432848203 2024-11-12T17:34:08,204 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e., pid=19, masterSystemTime=1731432848182 2024-11-12T17:34:08,207 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. 2024-11-12T17:34:08,207 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. 2024-11-12T17:34:08,208 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=a2efbf6b09e33aa73d99cb054d7d7f3e, regionState=OPEN, openSeqNum=2, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:08,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2efbf6b09e33aa73d99cb054d7d7f3e, server=1f32af3374c6,39493,1731432832063 because future has completed 2024-11-12T17:34:08,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-12T17:34:08,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure a2efbf6b09e33aa73d99cb054d7d7f3e, server=1f32af3374c6,39493,1731432832063 in 183 msec 2024-11-12T17:34:08,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-12T17:34:08,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns4:test-17314328406863, region=a2efbf6b09e33aa73d99cb054d7d7f3e, ASSIGN in 344 msec 2024-11-12T17:34:08,219 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns4:test-17314328406863 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T17:34:08,219 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17314328406863","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432848219"}]},"ts":"1731432848219"} 2024-11-12T17:34:08,221 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17314328406863, state=ENABLED in hbase:meta 2024-11-12T17:34:08,222 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17314328406863 execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T17:34:08,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns4:test-17314328406863 in 1.1880 sec 2024-11-12T17:34:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T17:34:09,182 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns4:test-17314328406863 completed 2024-11-12T17:34:09,182 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns4:test-17314328406863 get assigned. Timeout = 60000ms 2024-11-12T17:34:09,182 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T17:34:09,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns4:test-17314328406863 assigned to meta. Checking AM states. 2024-11-12T17:34:09,187 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T17:34:09,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns4:test-17314328406863 assigned. 2024-11-12T17:34:09,187 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T17:34:09,188 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestBackupBase.createTables(TestBackupBase.java:481) at org.apache.hadoop.hbase.backup.TestBackupBase.setUpHelper(TestBackupBase.java:343) at org.apache.hadoop.hbase.backup.TestBackupBase.setUp(TestBackupBase.java:357) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T17:34:09,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:09,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:09,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:34:09,222 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=610, OpenFileDescriptor=633, MaxFileDescriptor=1048576, SystemLoadAverage=185, ProcessCount=13, AvailableMemoryMB=9233 2024-11-12T17:34:09,224 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=610 is superior to 500 2024-11-12T17:34:09,225 INFO [Time-limited test {}] backup.TestIncrementalBackup(91): create full backup image for all tables 2024-11-12T17:34:09,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1731432840686 from 'test-1731432840686', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1731432840686', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:34:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=20, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1731432840686 2024-11-12T17:34:09,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741847_1023 (size=908) 2024-11-12T17:34:09,659 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/.tabledesc/.tableinfo.0000000001.367 2024-11-12T17:34:09,660 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/.tabledesc/.tableinfo.0000000002.908 2024-11-12T17:34:09,665 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1731432840686}] 2024-11-12T17:34:09,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, REOPEN/MOVE}] 2024-11-12T17:34:09,675 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, REOPEN/MOVE 2024-11-12T17:34:09,677 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=f14197f5e90bb7bd826d4a3ae95e1d7e, regionState=CLOSING, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:09,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, REOPEN/MOVE because future has completed 2024-11-12T17:34:09,680 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T17:34:09,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e, server=1f32af3374c6,39493,1731432832063}] 2024-11-12T17:34:09,838 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(122): Close f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:09,838 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T17:34:09,839 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1722): Closing f14197f5e90bb7bd826d4a3ae95e1d7e, disabling compactions & flushes 2024-11-12T17:34:09,839 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1755): Closing region test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:09,839 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1776): Time limited wait for close lock on test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:09,839 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1843): Acquired close lock on test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. after waiting 0 ms 2024-11-12T17:34:09,839 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1853): Updates disabled for region test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:09,843 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(2902): Flushing f14197f5e90bb7bd826d4a3ae95e1d7e 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-12T17:34:09,909 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/f/13333c2c3c1a42179067d2c31f89d94b is 37, key is row10/f:q1/1731432842388/Put/seqid=0 2024-11-12T17:34:09,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741848_1024 (size=8413) 2024-11-12T17:34:10,320 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/f/13333c2c3c1a42179067d2c31f89d94b 2024-11-12T17:34:10,381 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/f/13333c2c3c1a42179067d2c31f89d94b as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/13333c2c3c1a42179067d2c31f89d94b 2024-11-12T17:34:10,390 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/13333c2c3c1a42179067d2c31f89d94b, entries=99, sequenceid=103, filesize=8.2 K 2024-11-12T17:34:10,396 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for f14197f5e90bb7bd826d4a3ae95e1d7e in 553ms, sequenceid=103, compaction requested=false 2024-11-12T17:34:10,403 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2024-11-12T17:34:10,406 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T17:34:10,406 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1973): Closed test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:10,406 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1676): Region close journal for f14197f5e90bb7bd826d4a3ae95e1d7e: Waiting for close lock at 1731432849839Running coprocessor pre-close hooks at 1731432849839Disabling compacts and flushes for region at 1731432849839Disabling writes for close at 1731432849839Obtaining lock to block concurrent updates at 1731432849843 (+4 ms)Preparing flush snapshotting stores in f14197f5e90bb7bd826d4a3ae95e1d7e at 1731432849843Finished memstore snapshotting test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e., syncing WAL and waiting on mvcc, flushsize=dataSize=3247, getHeapSize=11248, getOffHeapSize=0, getCellsCount=99 at 1731432849851 (+8 ms)Flushing stores of test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. at 1731432849852 (+1 ms)Flushing f14197f5e90bb7bd826d4a3ae95e1d7e/f: creating writer at 1731432849854 (+2 ms)Flushing f14197f5e90bb7bd826d4a3ae95e1d7e/f: appending metadata at 1731432849898 (+44 ms)Flushing f14197f5e90bb7bd826d4a3ae95e1d7e/f: closing flushed file at 1731432849901 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15877b00: reopening flushed file at 1731432850379 (+478 ms)Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for f14197f5e90bb7bd826d4a3ae95e1d7e in 553ms, sequenceid=103, compaction requested=false at 1731432850397 (+18 ms)Writing region close event to WAL at 1731432850398 (+1 ms)Running coprocessor post-close hooks at 1731432850403 (+5 ms)Closed at 1731432850406 (+3 ms) 2024-11-12T17:34:10,406 WARN [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionServer(3299): Not adding moved region record: f14197f5e90bb7bd826d4a3ae95e1d7e to self. 2024-11-12T17:34:10,410 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(157): Closed f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:10,410 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=f14197f5e90bb7bd826d4a3ae95e1d7e, regionState=CLOSED 2024-11-12T17:34:10,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e, server=1f32af3374c6,39493,1731432832063 because future has completed 2024-11-12T17:34:10,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-12T17:34:10,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; CloseRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e, server=1f32af3374c6,39493,1731432832063 in 734 msec 2024-11-12T17:34:10,419 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, REOPEN/MOVE; state=CLOSED, location=1f32af3374c6,39493,1731432832063; forceNewPlan=false, retain=true 2024-11-12T17:34:10,570 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=f14197f5e90bb7bd826d4a3ae95e1d7e, regionState=OPENING, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:10,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, REOPEN/MOVE because future has completed 2024-11-12T17:34:10,574 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e, server=1f32af3374c6,39493,1731432832063}] 2024-11-12T17:34:10,697 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17314328406863 2024-11-12T17:34:10,697 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17314328406863 Metrics about Tables on a single HBase RegionServer 2024-11-12T17:34:10,698 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17314328406862 2024-11-12T17:34:10,698 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17314328406862 Metrics about Tables on a single HBase RegionServer 2024-11-12T17:34:10,699 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17314328406861 2024-11-12T17:34:10,699 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17314328406861 Metrics about Tables on a single HBase RegionServer 2024-11-12T17:34:10,735 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:10,735 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => f14197f5e90bb7bd826d4a3ae95e1d7e, NAME => 'test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e.', STARTKEY => '', ENDKEY => ''} 2024-11-12T17:34:10,736 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T17:34:10,736 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1731432840686 f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:10,736 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:34:10,736 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:10,736 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:10,738 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:10,740 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f14197f5e90bb7bd826d4a3ae95e1d7e columnFamilyName f 2024-11-12T17:34:10,740 DEBUG [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:10,752 DEBUG [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/13333c2c3c1a42179067d2c31f89d94b 2024-11-12T17:34:10,752 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.HStore(327): Store=f14197f5e90bb7bd826d4a3ae95e1d7e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:34:10,752 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:10,754 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f14197f5e90bb7bd826d4a3ae95e1d7e columnFamilyName f3 2024-11-12T17:34:10,754 DEBUG [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:10,755 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.HStore(327): Store=f14197f5e90bb7bd826d4a3ae95e1d7e/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:34:10,757 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:10,765 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f14197f5e90bb7bd826d4a3ae95e1d7e columnFamilyName mob 2024-11-12T17:34:10,776 DEBUG [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:10,777 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.HStore(327): Store=f14197f5e90bb7bd826d4a3ae95e1d7e/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:34:10,778 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:10,779 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:10,781 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:10,782 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:10,782 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:10,784 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1731432840686 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T17:34:10,786 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:10,787 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened f14197f5e90bb7bd826d4a3ae95e1d7e; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9896470080, jitterRate=-0.07831940054893494}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T17:34:10,787 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:10,788 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for f14197f5e90bb7bd826d4a3ae95e1d7e: Running coprocessor pre-open hook at 1731432850736Writing region info on filesystem at 1731432850736Initializing all the Stores at 1731432850738 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432850738Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432850738Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432850738Cleaning up temporary data from old regions at 1731432850782 (+44 ms)Running coprocessor post-open hooks at 1731432850787 (+5 ms)Region opened successfully at 1731432850788 (+1 ms) 2024-11-12T17:34:10,790 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e., pid=24, masterSystemTime=1731432850727 2024-11-12T17:34:10,793 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:10,793 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:10,794 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=f14197f5e90bb7bd826d4a3ae95e1d7e, regionState=OPEN, openSeqNum=107, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:10,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e, server=1f32af3374c6,39493,1731432832063 because future has completed 2024-11-12T17:34:10,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22 2024-11-12T17:34:10,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e, server=1f32af3374c6,39493,1731432832063 in 224 msec 2024-11-12T17:34:10,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-12T17:34:10,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, REOPEN/MOVE in 1.1270 sec 2024-11-12T17:34:10,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-12T17:34:10,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1731432840686 in 1.1410 sec 2024-11-12T17:34:10,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1731432840686 in 1.5770 sec 2024-11-12T17:34:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=20 2024-11-12T17:34:10,817 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1731432840686 completed 2024-11-12T17:34:10,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28393699, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:10,818 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:10,818 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:10,820 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:10,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:10,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:10,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e3a4d48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:10,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:10,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:10,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:10,822 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:10,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10c75cfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:10,825 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:10,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:10,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:10,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:10,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1731432840686', row='row-test-1731432840686-3-0', locateType=CURRENT is [region=test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e., hostname=1f32af3374c6,39493,1731432832063, seqNum=107] 2024-11-12T17:34:10,902 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T17:34:10,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 1f32af3374c6,39475,1731432830222 2024-11-12T17:34:10,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1896e074 2024-11-12T17:34:10,908 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T17:34:10,910 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T17:34:10,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'backup'} 2024-11-12T17:34:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=backup 2024-11-12T17:34:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T17:34:10,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=backup in 11 msec 2024-11-12T17:34:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T17:34:11,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: backup completed 2024-11-12T17:34:11,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:34:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system 2024-11-12T17:34:11,066 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T17:34:11,066 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:11,066 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2024-11-12T17:34:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T17:34:11,069 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T17:34:11,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741849_1025 (size=635) 2024-11-12T17:34:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T17:34:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T17:34:11,485 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a91481e8504b1563b1463c687c54efd, NAME => 'backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 2024-11-12T17:34:11,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741850_1026 (size=40) 2024-11-12T17:34:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T17:34:11,894 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:34:11,894 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1722): Closing 1a91481e8504b1563b1463c687c54efd, disabling compactions & flushes 2024-11-12T17:34:11,894 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:34:11,894 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:34:11,894 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. after waiting 0 ms 2024-11-12T17:34:11,894 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:34:11,894 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1973): Closed backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:34:11,895 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a91481e8504b1563b1463c687c54efd: Waiting for close lock at 1731432851894Disabling compacts and flushes for region at 1731432851894Disabling writes for close at 1731432851894Writing region close event to WAL at 1731432851894Closed at 1731432851894 2024-11-12T17:34:11,897 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T17:34:11,897 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1731432851897"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731432851897"}]},"ts":"1731432851897"} 2024-11-12T17:34:11,900 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T17:34:11,901 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T17:34:11,902 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432851902"}]},"ts":"1731432851902"} 2024-11-12T17:34:11,904 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-12T17:34:11,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=1a91481e8504b1563b1463c687c54efd, ASSIGN}] 2024-11-12T17:34:11,906 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=1a91481e8504b1563b1463c687c54efd, ASSIGN 2024-11-12T17:34:11,907 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=1a91481e8504b1563b1463c687c54efd, ASSIGN; state=OFFLINE, location=1f32af3374c6,39493,1731432832063; forceNewPlan=false, retain=false 2024-11-12T17:34:12,058 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1a91481e8504b1563b1463c687c54efd, regionState=OPENING, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:12,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=1a91481e8504b1563b1463c687c54efd, ASSIGN because future has completed 2024-11-12T17:34:12,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a91481e8504b1563b1463c687c54efd, server=1f32af3374c6,39493,1731432832063}] 2024-11-12T17:34:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T17:34:12,219 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:34:12,220 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a91481e8504b1563b1463c687c54efd, NAME => 'backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.', STARTKEY => '', ENDKEY => ''} 2024-11-12T17:34:12,221 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T17:34:12,221 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:12,221 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:34:12,221 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:12,221 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:12,224 INFO [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:12,226 INFO [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a91481e8504b1563b1463c687c54efd columnFamilyName meta 2024-11-12T17:34:12,226 DEBUG [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:12,226 INFO [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] regionserver.HStore(327): Store=1a91481e8504b1563b1463c687c54efd/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:34:12,227 INFO [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:12,228 INFO [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a91481e8504b1563b1463c687c54efd columnFamilyName session 2024-11-12T17:34:12,229 DEBUG [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:12,229 INFO [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] regionserver.HStore(327): Store=1a91481e8504b1563b1463c687c54efd/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:34:12,229 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:12,230 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:12,231 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:12,232 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:12,232 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:12,234 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-12T17:34:12,235 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:12,238 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T17:34:12,239 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 1a91481e8504b1563b1463c687c54efd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11755411520, jitterRate=0.09480801224708557}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-12T17:34:12,239 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:12,240 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 1a91481e8504b1563b1463c687c54efd: Running coprocessor pre-open hook at 1731432852221Writing region info on filesystem at 1731432852221Initializing all the Stores at 1731432852223 (+2 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432852223Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432852223Cleaning up temporary data from old regions at 1731432852232 (+9 ms)Running coprocessor post-open hooks at 1731432852240 (+8 ms)Region opened successfully at 1731432852240 2024-11-12T17:34:12,242 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd., pid=28, masterSystemTime=1731432852214 2024-11-12T17:34:12,245 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:34:12,245 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:34:12,246 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1a91481e8504b1563b1463c687c54efd, regionState=OPEN, openSeqNum=2, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:12,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a91481e8504b1563b1463c687c54efd, server=1f32af3374c6,39493,1731432832063 because future has completed 2024-11-12T17:34:12,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-12T17:34:12,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a91481e8504b1563b1463c687c54efd, server=1f32af3374c6,39493,1731432832063 in 189 msec 2024-11-12T17:34:12,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-12T17:34:12,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=1a91481e8504b1563b1463c687c54efd, ASSIGN in 348 msec 2024-11-12T17:34:12,257 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T17:34:12,257 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432852257"}]},"ts":"1731432852257"} 2024-11-12T17:34:12,260 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-12T17:34:12,261 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T17:34:12,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system in 1.1990 sec 2024-11-12T17:34:12,773 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17314328406863' 2024-11-12T17:34:12,774 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-11-12T17:34:13,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T17:34:13,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system completed 2024-11-12T17:34:13,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:34:13,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system_bulk 2024-11-12T17:34:13,234 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T17:34:13,234 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:13,234 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2024-11-12T17:34:13,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T17:34:13,235 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T17:34:13,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741851_1027 (size=640) 2024-11-12T17:34:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T17:34:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T17:34:13,646 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 569aa6aeafd4b9d079658f569537f60c, NAME => 'backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 2024-11-12T17:34:13,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741852_1028 (size=45) 2024-11-12T17:34:13,655 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:34:13,655 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1722): Closing 569aa6aeafd4b9d079658f569537f60c, disabling compactions & flushes 2024-11-12T17:34:13,655 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. 2024-11-12T17:34:13,655 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. 2024-11-12T17:34:13,655 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. after waiting 0 ms 2024-11-12T17:34:13,655 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. 2024-11-12T17:34:13,655 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1973): Closed backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. 2024-11-12T17:34:13,656 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1676): Region close journal for 569aa6aeafd4b9d079658f569537f60c: Waiting for close lock at 1731432853655Disabling compacts and flushes for region at 1731432853655Disabling writes for close at 1731432853655Writing region close event to WAL at 1731432853655Closed at 1731432853655 2024-11-12T17:34:13,657 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T17:34:13,658 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731432853658"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731432853658"}]},"ts":"1731432853658"} 2024-11-12T17:34:13,661 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T17:34:13,662 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T17:34:13,662 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432853662"}]},"ts":"1731432853662"} 2024-11-12T17:34:13,664 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2024-11-12T17:34:13,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=569aa6aeafd4b9d079658f569537f60c, ASSIGN}] 2024-11-12T17:34:13,666 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=569aa6aeafd4b9d079658f569537f60c, ASSIGN 2024-11-12T17:34:13,667 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=569aa6aeafd4b9d079658f569537f60c, ASSIGN; state=OFFLINE, location=1f32af3374c6,39493,1731432832063; forceNewPlan=false, retain=false 2024-11-12T17:34:13,690 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-12T17:34:13,818 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=569aa6aeafd4b9d079658f569537f60c, regionState=OPENING, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:13,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=569aa6aeafd4b9d079658f569537f60c, ASSIGN because future has completed 2024-11-12T17:34:13,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 569aa6aeafd4b9d079658f569537f60c, server=1f32af3374c6,39493,1731432832063}] 2024-11-12T17:34:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T17:34:13,979 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. 2024-11-12T17:34:13,980 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 569aa6aeafd4b9d079658f569537f60c, NAME => 'backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c.', STARTKEY => '', ENDKEY => ''} 2024-11-12T17:34:13,980 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T17:34:13,980 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:34:13,981 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:34:13,981 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:34:13,981 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:34:13,983 INFO [StoreOpener-569aa6aeafd4b9d079658f569537f60c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:34:13,985 INFO [StoreOpener-569aa6aeafd4b9d079658f569537f60c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 569aa6aeafd4b9d079658f569537f60c columnFamilyName meta 2024-11-12T17:34:13,985 DEBUG [StoreOpener-569aa6aeafd4b9d079658f569537f60c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:13,985 INFO [StoreOpener-569aa6aeafd4b9d079658f569537f60c-1 {}] regionserver.HStore(327): Store=569aa6aeafd4b9d079658f569537f60c/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:34:13,986 INFO [StoreOpener-569aa6aeafd4b9d079658f569537f60c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:34:13,987 INFO [StoreOpener-569aa6aeafd4b9d079658f569537f60c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 569aa6aeafd4b9d079658f569537f60c columnFamilyName session 2024-11-12T17:34:13,987 DEBUG [StoreOpener-569aa6aeafd4b9d079658f569537f60c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:13,988 INFO [StoreOpener-569aa6aeafd4b9d079658f569537f60c-1 {}] regionserver.HStore(327): Store=569aa6aeafd4b9d079658f569537f60c/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:34:13,988 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:34:13,989 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system_bulk/569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:34:13,990 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system_bulk/569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:34:13,991 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:34:13,991 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:34:13,992 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-12T17:34:13,994 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:34:13,997 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system_bulk/569aa6aeafd4b9d079658f569537f60c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T17:34:13,998 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 569aa6aeafd4b9d079658f569537f60c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11390420160, jitterRate=0.06081554293632507}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-12T17:34:13,998 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:34:13,999 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 569aa6aeafd4b9d079658f569537f60c: Running coprocessor pre-open hook at 1731432853981Writing region info on filesystem at 1731432853981Initializing all the Stores at 1731432853982 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432853982Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432853982Cleaning up temporary data from old regions at 1731432853991 (+9 ms)Running coprocessor post-open hooks at 1731432853998 (+7 ms)Region opened successfully at 1731432853999 (+1 ms) 2024-11-12T17:34:14,000 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c., pid=31, masterSystemTime=1731432853974 2024-11-12T17:34:14,003 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. 2024-11-12T17:34:14,003 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. 2024-11-12T17:34:14,004 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=569aa6aeafd4b9d079658f569537f60c, regionState=OPEN, openSeqNum=2, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:14,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 569aa6aeafd4b9d079658f569537f60c, server=1f32af3374c6,39493,1731432832063 because future has completed 2024-11-12T17:34:14,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-12T17:34:14,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 569aa6aeafd4b9d079658f569537f60c, server=1f32af3374c6,39493,1731432832063 in 186 msec 2024-11-12T17:34:14,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-12T17:34:14,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=569aa6aeafd4b9d079658f569537f60c, ASSIGN in 346 msec 2024-11-12T17:34:14,015 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T17:34:14,016 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432854015"}]},"ts":"1731432854015"} 2024-11-12T17:34:14,018 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2024-11-12T17:34:14,019 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T17:34:14,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system_bulk in 788 msec 2024-11-12T17:34:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T17:34:14,372 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system_bulk completed 2024-11-12T17:34:14,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:34:14,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:34:14,398 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-12T17:34:14,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='activesession:', locateType=CURRENT is [region=backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd., hostname=1f32af3374c6,39493,1731432832063, seqNum=2] 2024-11-12T17:34:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39493 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2024-11-12T17:34:14,424 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T17:34:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731432854424 (current time:1731432854424). 2024-11-12T17:34:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T17:34:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-12T17:34:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd90004, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:14,426 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:14,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:14,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:14,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b2ccb75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:14,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:14,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:14,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:14,429 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:14,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@661162e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:14,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:14,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:14,435 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:14,437 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475. 2024-11-12T17:34:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T17:34:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:14,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:34:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T17:34:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T17:34:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-12T17:34:14,450 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T17:34:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T17:34:14,454 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T17:34:14,470 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T17:34:14,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741853_1029 (size=52) 2024-11-12T17:34:14,479 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T17:34:14,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd}] 2024-11-12T17:34:14,486 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:14,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T17:34:14,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39493 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-11-12T17:34:14,645 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:34:14,646 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 1a91481e8504b1563b1463c687c54efd 2/2 column families, dataSize=45 B heapSize=632 B 2024-11-12T17:34:14,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/0fc13f1a42ee47e4b896a898f018a2c6 is 49, key is activesession:/session:c/1731432854413/Put/seqid=0 2024-11-12T17:34:14,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741854_1030 (size=5049) 2024-11-12T17:34:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T17:34:15,079 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/0fc13f1a42ee47e4b896a898f018a2c6 2024-11-12T17:34:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T17:34:15,090 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/0fc13f1a42ee47e4b896a898f018a2c6 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6 2024-11-12T17:34:15,099 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6, entries=1, sequenceid=5, filesize=4.9 K 2024-11-12T17:34:15,100 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 1a91481e8504b1563b1463c687c54efd in 454ms, sequenceid=5, compaction requested=false 2024-11-12T17:34:15,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 1a91481e8504b1563b1463c687c54efd: 2024-11-12T17:34:15,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. for snapshot_backup_system completed. 2024-11-12T17:34:15,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.' region-info for snapshot=snapshot_backup_system 2024-11-12T17:34:15,105 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T17:34:15,108 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-12T17:34:15,108 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6] hfiles 2024-11-12T17:34:15,110 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6 for snapshot=snapshot_backup_system 2024-11-12T17:34:15,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741855_1031 (size=95) 2024-11-12T17:34:15,522 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:34:15,524 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-12T17:34:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-12T17:34:15,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:15,527 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:15,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-12T17:34:15,530 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T17:34:15,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd in 1.0460 sec 2024-11-12T17:34:15,532 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T17:34:15,533 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T17:34:15,533 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-12T17:34:15,536 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-12T17:34:15,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741856_1032 (size=731) 2024-11-12T17:34:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T17:34:15,954 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T17:34:15,970 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T17:34:15,972 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_backup_system 2024-11-12T17:34:15,974 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T17:34:15,974 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-12T17:34:15,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.5300 sec 2024-11-12T17:34:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T17:34:16,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-12T17:34:16,603 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1731432850879 started at 1731432856602. 2024-11-12T17:34:16,617 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1731432850879 has been started. 2024-11-12T17:34:16,622 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-12T17:34:16,628 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-12T17:34:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-12T17:34:16,631 INFO [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-12T17:34:16,631 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-12T17:34:16,631 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-12T17:34:16,631 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-12T17:34:16,632 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T17:34:16,632 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T17:34:16,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T17:34:16,634 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:16,634 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T17:34:16,635 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T17:34:16,635 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:16,635 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-12T17:34:16,635 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-12T17:34:16,636 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T17:34:16,636 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-12T17:34:16,636 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T17:34:16,636 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-12T17:34:16,638 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-12T17:34:16,638 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-12T17:34:16,639 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-12T17:34:16,639 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-12T17:34:16,639 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-12T17:34:16,639 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-12T17:34:16,639 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '1f32af3374c6,39493,1731432832063' joining acquired barrier for procedure (rolllog) in zk 2024-11-12T17:34:16,641 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:16,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:16,641 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:16,641 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T17:34:16,642 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T17:34:16,642 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-12T17:34:16,642 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-12T17:34:16,642 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T17:34:16,643 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T17:34:16,643 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:16,643 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:34:16,644 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T17:34:16,644 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '1f32af3374c6,39493,1731432832063' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-12T17:34:16,644 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6e5e676[Count = 0] remaining members to acquire global barrier 2024-11-12T17:34:16,644 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-12T17:34:16,645 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:16,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:16,646 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:16,646 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:16,646 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-12T17:34:16,647 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:16,647 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-12T17:34:16,647 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-12T17:34:16,647 DEBUG [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 1f32af3374c6,39493,1731432832063 2024-11-12T17:34:16,654 INFO [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1731432834619 highest: 1731432842004 on 1f32af3374c6,39493,1731432832063 2024-11-12T17:34:16,655 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0:(num 1731432834619) roll requested 2024-11-12T17:34:16,677 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432856655, exclude list is [], retry=0 2024-11-12T17:34:16,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:34:16,683 INFO [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432834619 with entries=3, filesize=1.00 KB; new WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432856655 2024-11-12T17:34:16,684 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:34:16,684 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432834619 is not closed yet, will try archiving it next time 2024-11-12T17:34:16,684 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 1f32af3374c6%2C39493%2C1731432832063.meta:.meta(num 1731432835143) roll requested 2024-11-12T17:34:16,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741832_1008 (size=1034) 2024-11-12T17:34:16,688 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432834619 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432834619 2024-11-12T17:34:16,708 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432856684.meta, exclude list is [], retry=0 2024-11-12T17:34:16,711 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:34:16,713 INFO [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432835143.meta with entries=42, filesize=11.76 KB; new WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432856684.meta 2024-11-12T17:34:16,714 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:34:16,714 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432835143.meta is not closed yet, will try archiving it next time 2024-11-12T17:34:16,714 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1:(num 1731432842004) roll requested 2024-11-12T17:34:16,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741834_1010 (size=12055) 2024-11-12T17:34:16,729 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432856714, exclude list is [], retry=0 2024-11-12T17:34:16,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:34:16,734 INFO [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432842004 with entries=24, filesize=4.64 KB; new WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432856714 2024-11-12T17:34:16,734 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:34:16,734 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432842004 is not closed yet, will try archiving it next time 2024-11-12T17:34:16,735 DEBUG [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 80 2024-11-12T17:34:16,736 INFO [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1731432856655 on 1f32af3374c6,39493,1731432832063 2024-11-12T17:34:16,736 DEBUG [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T17:34:16,736 DEBUG [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] client.AsyncConnectionImpl(321): The fetched master address is 1f32af3374c6,39475,1731432830222 2024-11-12T17:34:16,736 DEBUG [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@45956c66 2024-11-12T17:34:16,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741840_1016 (size=4756) 2024-11-12T17:34:16,737 DEBUG [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T17:34:16,738 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50173, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-12T17:34:16,739 DEBUG [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:16,739 DEBUG [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:16,739 DEBUG [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:16,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46051, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-12T17:34:16,750 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:34:16,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:34:16,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='rslogts:hdfs://localhost:32879/backupUT\x00', locateType=CURRENT is [region=backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd., hostname=1f32af3374c6,39493,1731432832063, seqNum=2] 2024-11-12T17:34:16,766 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-12T17:34:16,766 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '1f32af3374c6,39493,1731432832063' in zk 2024-11-12T17:34:16,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:16,919 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-12T17:34:16,919 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T17:34:16,919 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:16,919 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T17:34:16,919 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T17:34:16,919 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-12T17:34:16,920 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T17:34:16,921 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T17:34:16,921 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:16,922 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:34:16,922 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T17:34:16,923 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:16,923 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:34:16,924 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '1f32af3374c6,39493,1731432832063': 2024-11-12T17:34:16,924 INFO [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-12T17:34:16,924 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-12T17:34:16,924 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-12T17:34:16,925 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-12T17:34:16,925 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '1f32af3374c6,39493,1731432832063' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-12T17:34:16,925 INFO [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-12T17:34:17,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T17:34:17,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T17:34:17,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T17:34:17,012 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-12T17:34:17,012 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-12T17:34:17,012 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T17:34:17,012 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T17:34:17,012 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T17:34:17,012 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T17:34:17,012 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T17:34:17,012 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T17:34:17,013 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:17,013 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:17,013 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T17:34:17,013 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T17:34:17,014 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:17,015 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:34:17,015 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:17,015 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T17:34:17,016 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:17,017 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:34:17,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:17,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T17:34:17,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T17:34:17,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T17:34:17,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T17:34:17,119 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T17:34:17,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:17,119 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T17:34:17,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:17,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:17,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T17:34:17,120 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T17:34:17,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-12T17:34:17,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-12T17:34:17,120 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T17:34:17,120 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T17:34:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-12T17:34:17,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1731432857137_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } 2024-11-12T17:34:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731432857138 (current time:1731432857138). 2024-11-12T17:34:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T17:34:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731432857137_default_test-1731432840686 VERSION not specified, setting to 2 2024-11-12T17:34:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@288b59bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:17,140 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:17,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:17,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:17,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be9d1a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:17,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:17,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:17,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:17,142 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:17,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33847882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:17,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:17,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:17,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:17,147 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475. 2024-11-12T17:34:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T17:34:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:17,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:34:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T17:34:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731432857137_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } 2024-11-12T17:34:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731432857137_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-12T17:34:17,150 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731432857137_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T17:34:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T17:34:17,152 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731432857137_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T17:34:17,154 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731432857137_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T17:34:17,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741860_1036 (size=84) 2024-11-12T17:34:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T17:34:17,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T17:34:17,562 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731432857137_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T17:34:17,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e}] 2024-11-12T17:34:17,563 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:17,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-11-12T17:34:17,716 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:17,717 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing f14197f5e90bb7bd826d4a3ae95e1d7e 3/3 column families, dataSize=654 B heapSize=2.25 KB 2024-11-12T17:34:17,735 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/f3/db6ea8024d2f42608b55815168d39424 is 58, key is row-test-1731432840686-3-0/f3:q1/1731432850833/Put/seqid=0 2024-11-12T17:34:17,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741861_1037 (size=5387) 2024-11-12T17:34:17,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T17:34:18,141 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/f3/db6ea8024d2f42608b55815168d39424 2024-11-12T17:34:18,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741862_1038 (size=4469) 2024-11-12T17:34:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T17:34:18,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/mob/5c0434f3a70a41059b32fac676fc1ec2 is 59, key is row-test-1731432840686-3-0/mob:q1/1731432850852/Put/seqid=0 2024-11-12T17:34:18,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741863_1039 (size=5484) 2024-11-12T17:34:18,775 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2024-11-12T17:34:18,994 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=330, hasBloomFilter=true, into tmp file hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/mob/5c0434f3a70a41059b32fac676fc1ec2 2024-11-12T17:34:19,003 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/f3/db6ea8024d2f42608b55815168d39424 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f3/db6ea8024d2f42608b55815168d39424 2024-11-12T17:34:19,011 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f3/db6ea8024d2f42608b55815168d39424, entries=6, sequenceid=121, filesize=5.3 K 2024-11-12T17:34:19,012 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/mob/5c0434f3a70a41059b32fac676fc1ec2 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/5c0434f3a70a41059b32fac676fc1ec2 2024-11-12T17:34:19,019 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/5c0434f3a70a41059b32fac676fc1ec2, entries=6, sequenceid=121, filesize=5.4 K 2024-11-12T17:34:19,021 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~654 B/654, heapSize ~1.97 KB/2016, currentSize=0 B/0 for f14197f5e90bb7bd826d4a3ae95e1d7e in 1304ms, sequenceid=121, compaction requested=false 2024-11-12T17:34:19,021 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for f14197f5e90bb7bd826d4a3ae95e1d7e: 2024-11-12T17:34:19,021 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. for snapshot_1731432857137_default_test-1731432840686 completed. 2024-11-12T17:34:19,021 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e.' region-info for snapshot=snapshot_1731432857137_default_test-1731432840686 2024-11-12T17:34:19,021 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T17:34:19,021 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/13333c2c3c1a42179067d2c31f89d94b] hfiles 2024-11-12T17:34:19,021 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/13333c2c3c1a42179067d2c31f89d94b for snapshot=snapshot_1731432857137_default_test-1731432840686 2024-11-12T17:34:19,021 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f3/db6ea8024d2f42608b55815168d39424] hfiles 2024-11-12T17:34:19,021 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f3/db6ea8024d2f42608b55815168d39424 for snapshot=snapshot_1731432857137_default_test-1731432840686 2024-11-12T17:34:19,021 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/5c0434f3a70a41059b32fac676fc1ec2] hfiles 2024-11-12T17:34:19,021 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/5c0434f3a70a41059b32fac676fc1ec2 for snapshot=snapshot_1731432857137_default_test-1731432840686 2024-11-12T17:34:19,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741864_1040 (size=185) 2024-11-12T17:34:19,070 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T17:34:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T17:34:19,428 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:19,428 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-12T17:34:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-12T17:34:19,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731432857137_default_test-1731432840686 on region f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:19,430 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:19,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-12T17:34:19,434 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731432857137_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T17:34:19,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; SnapshotRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e in 1.8690 sec 2024-11-12T17:34:19,435 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731432857137_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T17:34:19,437 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-12T17:34:19,437 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-12T17:34:19,437 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:19,438 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-12T17:34:19,438 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:19,439 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f3 2024-11-12T17:34:19,439 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:19,439 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: mob 2024-11-12T17:34:19,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741865_1041 (size=49) 2024-11-12T17:34:19,483 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-12T17:34:19,848 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731432857137_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T17:34:19,848 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731432857137_default_test-1731432840686 2024-11-12T17:34:19,849 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_1731432857137_default_test-1731432840686 2024-11-12T17:34:19,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741866_1042 (size=1146) 2024-11-12T17:34:19,863 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731432857137_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T17:34:19,872 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731432857137_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T17:34:19,873 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_1731432857137_default_test-1731432840686 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_1731432857137_default_test-1731432840686 2024-11-12T17:34:19,875 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731432857137_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T17:34:19,875 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731432857137_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-12T17:34:19,877 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731432857137_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } in 2.7270 sec 2024-11-12T17:34:20,697 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-12T17:34:20,697 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2024-11-12T17:34:20,698 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-11-12T17:34:20,698 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2024-11-12T17:34:20,698 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1731432840686 2024-11-12T17:34:20,698 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1731432840686 Metrics about Tables on a single HBase RegionServer 2024-11-12T17:34:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T17:34:21,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1731432840686 completed 2024-11-12T17:34:21,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1731432861312_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } 2024-11-12T17:34:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731432861313 (current time:1731432861313). 2024-11-12T17:34:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T17:34:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731432861312_ns2_test-17314328406861 VERSION not specified, setting to 2 2024-11-12T17:34:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a23b0d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:21,314 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:21,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:21,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:21,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@669e749e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:21,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:21,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:21,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:21,316 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:21,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3240b72f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:21,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:21,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:21,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:21,320 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:21,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475. 2024-11-12T17:34:21,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T17:34:21,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:21,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:21,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:34:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T17:34:21,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731432861312_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } 2024-11-12T17:34:21,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731432861312_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-12T17:34:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T17:34:21,326 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731432861312_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T17:34:21,327 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731432861312_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T17:34:21,330 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731432861312_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T17:34:21,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741867_1043 (size=86) 2024-11-12T17:34:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T17:34:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T17:34:21,738 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731432861312_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T17:34:21,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6dd38527fc6c5b38bfed421a5b5be067}] 2024-11-12T17:34:21,740 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:21,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2024-11-12T17:34:21,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:34:21,893 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 6dd38527fc6c5b38bfed421a5b5be067 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-12T17:34:21,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/.tmp/f/9f1293a0d2294237898b5e4d02393945 is 37, key is row10/f:q1/1731432844755/Put/seqid=0 2024-11-12T17:34:21,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741868_1044 (size=8413) 2024-11-12T17:34:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T17:34:22,320 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/.tmp/f/9f1293a0d2294237898b5e4d02393945 2024-11-12T17:34:22,330 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/.tmp/f/9f1293a0d2294237898b5e4d02393945 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/9f1293a0d2294237898b5e4d02393945 2024-11-12T17:34:22,337 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/9f1293a0d2294237898b5e4d02393945, entries=99, sequenceid=103, filesize=8.2 K 2024-11-12T17:34:22,339 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 6dd38527fc6c5b38bfed421a5b5be067 in 445ms, sequenceid=103, compaction requested=false 2024-11-12T17:34:22,339 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 6dd38527fc6c5b38bfed421a5b5be067: 2024-11-12T17:34:22,339 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. for snapshot_1731432861312_ns2_test-17314328406861 completed. 2024-11-12T17:34:22,339 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067.' region-info for snapshot=snapshot_1731432861312_ns2_test-17314328406861 2024-11-12T17:34:22,339 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T17:34:22,339 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/9f1293a0d2294237898b5e4d02393945] hfiles 2024-11-12T17:34:22,339 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/9f1293a0d2294237898b5e4d02393945 for snapshot=snapshot_1731432861312_ns2_test-17314328406861 2024-11-12T17:34:22,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741869_1045 (size=91) 2024-11-12T17:34:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T17:34:22,746 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:34:22,746 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-12T17:34:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-12T17:34:22,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731432861312_ns2_test-17314328406861 on region 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:22,748 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:22,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-12T17:34:22,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 6dd38527fc6c5b38bfed421a5b5be067 in 1.0110 sec 2024-11-12T17:34:22,751 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731432861312_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T17:34:22,752 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731432861312_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T17:34:22,753 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731432861312_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T17:34:22,753 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731432861312_ns2_test-17314328406861 2024-11-12T17:34:22,754 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_1731432861312_ns2_test-17314328406861 2024-11-12T17:34:22,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741870_1046 (size=456) 2024-11-12T17:34:23,167 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731432861312_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T17:34:23,174 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731432861312_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T17:34:23,175 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_1731432861312_ns2_test-17314328406861 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_1731432861312_ns2_test-17314328406861 2024-11-12T17:34:23,176 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731432861312_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T17:34:23,176 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731432861312_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-12T17:34:23,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731432861312_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } in 1.8530 sec 2024-11-12T17:34:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T17:34:23,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17314328406861 completed 2024-11-12T17:34:23,472 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1731432850879 2024-11-12T17:34:23,472 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-12T17:34:23,476 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2024-11-12T17:34:23,481 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731432850879_ns2:test-17314328406861 2024-11-12T17:34:23,481 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731432861312_ns2_test-17314328406861 to hdfs://localhost:32879/backupUT/backup_1731432850879/ns2/test-17314328406861/ 2024-11-12T17:34:23,486 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-12T17:34:23,521 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:32879, inputRoot=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 2024-11-12T17:34:23,521 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1613456041_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:32879/backupUT/backup_1731432850879/ns2/test-17314328406861, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:32879/backupUT/backup_1731432850879/ns2/test-17314328406861/.hbase-snapshot/.tmp/snapshot_1731432861312_ns2_test-17314328406861 2024-11-12T17:34:23,524 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-12T17:34:23,530 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_1731432861312_ns2_test-17314328406861 to hdfs://localhost:32879/backupUT/backup_1731432850879/ns2/test-17314328406861/.hbase-snapshot/.tmp/snapshot_1731432861312_ns2_test-17314328406861 2024-11-12T17:34:23,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741871_1047 (size=456) 2024-11-12T17:34:23,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741872_1048 (size=86) 2024-11-12T17:34:23,953 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:23,953 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:23,953 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:24,992 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop-8336810705935853136.jar 2024-11-12T17:34:24,992 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:24,993 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:24,993 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:24,994 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:24,994 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:24,994 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:24,995 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:24,995 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:24,996 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:24,996 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T17:34:24,996 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T17:34:24,997 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T17:34:24,997 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T17:34:24,997 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T17:34:24,998 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T17:34:24,998 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T17:34:24,998 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T17:34:24,999 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T17:34:24,999 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T17:34:24,999 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T17:34:25,002 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:34:25,002 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:34:25,003 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:34:25,003 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:34:25,003 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:34:25,003 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:34:25,004 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:34:25,072 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T17:34:25,104 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731432861312_ns2_test-17314328406861' hfile list 2024-11-12T17:34:25,111 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=8.2 K 2024-11-12T17:34:25,327 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local611249186_0001_51d6e4e8-ea20-4c67-9d82-f60f0dfceb46/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-12T17:34:25,328 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local611249186_0001_51d6e4e8-ea20-4c67-9d82-f60f0dfceb46/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-12T17:34:25,489 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-12T17:34:25,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741873_1049 (size=8413) 2024-11-12T17:34:26,001 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/9f1293a0d2294237898b5e4d02393945 output=hdfs://localhost:32879/backupUT/backup_1731432850879/ns2/test-17314328406861/archive/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/9f1293a0d2294237898b5e4d02393945 2024-11-12T17:34:26,003 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.016M/sec 2024-11-12T17:34:26,407 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-12T17:34:26,408 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-12T17:34:26,415 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731432861312_ns2_test-17314328406861 2024-11-12T17:34:26,415 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731432861312_ns2_test-17314328406861 finished. 2024-11-12T17:34:26,415 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731432850879_test-1731432840686 2024-11-12T17:34:26,415 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731432857137_default_test-1731432840686 to hdfs://localhost:32879/backupUT/backup_1731432850879/default/test-1731432840686/ 2024-11-12T17:34:26,415 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-12T17:34:26,447 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:32879, inputRoot=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 2024-11-12T17:34:26,447 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1613456041_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:32879/backupUT/backup_1731432850879/default/test-1731432840686, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:32879/backupUT/backup_1731432850879/default/test-1731432840686/.hbase-snapshot/.tmp/snapshot_1731432857137_default_test-1731432840686 2024-11-12T17:34:26,449 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-12T17:34:26,456 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_1731432857137_default_test-1731432840686 to hdfs://localhost:32879/backupUT/backup_1731432850879/default/test-1731432840686/.hbase-snapshot/.tmp/snapshot_1731432857137_default_test-1731432840686 2024-11-12T17:34:26,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741874_1050 (size=84) 2024-11-12T17:34:26,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741875_1051 (size=1146) 2024-11-12T17:34:26,870 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:26,870 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:26,871 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:27,875 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop-8721105796355880519.jar 2024-11-12T17:34:27,875 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:27,876 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:27,876 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:27,876 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:27,877 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:27,877 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:27,877 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:27,878 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:27,878 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:27,878 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T17:34:27,878 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T17:34:27,879 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T17:34:27,879 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T17:34:27,880 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T17:34:27,880 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T17:34:27,880 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T17:34:27,881 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T17:34:27,881 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T17:34:27,881 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T17:34:27,881 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T17:34:27,882 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:34:27,882 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:34:27,882 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:34:27,883 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:34:27,883 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:34:27,883 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:34:27,884 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:34:27,927 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T17:34:27,955 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731432857137_default_test-1731432840686' hfile list 2024-11-12T17:34:27,958 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=18.8 K 2024-11-12T17:34:28,052 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local1702154611_0002_9067a346-96d4-4629-84a0-4a08fae5fb2c/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-12T17:34:28,052 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local1702154611_0002_9067a346-96d4-4629-84a0-4a08fae5fb2c/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-12T17:34:28,134 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-12T17:34:28,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741876_1052 (size=8413) 2024-11-12T17:34:28,579 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/13333c2c3c1a42179067d2c31f89d94b output=hdfs://localhost:32879/backupUT/backup_1731432850879/default/test-1731432840686/archive/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/13333c2c3c1a42179067d2c31f89d94b 2024-11-12T17:34:28,579 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-12T17:34:28,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741877_1053 (size=5484) 2024-11-12T17:34:29,051 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/5c0434f3a70a41059b32fac676fc1ec2 output=hdfs://localhost:32879/backupUT/backup_1731432850879/default/test-1731432840686/archive/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/5c0434f3a70a41059b32fac676fc1ec2 2024-11-12T17:34:29,051 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5484 (5.4 K) time=0sec 0.011M/sec 2024-11-12T17:34:29,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741878_1054 (size=5387) 2024-11-12T17:34:29,507 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f3/db6ea8024d2f42608b55815168d39424 output=hdfs://localhost:32879/backupUT/backup_1731432850879/default/test-1731432840686/archive/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f3/db6ea8024d2f42608b55815168d39424 2024-11-12T17:34:29,508 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5387 (5.3 K) time=0sec 0.011M/sec 2024-11-12T17:34:30,103 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-12T17:34:30,105 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-12T17:34:30,115 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731432857137_default_test-1731432840686 2024-11-12T17:34:30,115 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731432857137_default_test-1731432840686 finished. 2024-11-12T17:34:30,116 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17314328406861 2024-11-12T17:34:30,116 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1731432840686 2024-11-12T17:34:30,179 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731432850879 2024-11-12T17:34:30,180 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-12T17:34:30,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741879_1055 (size=130) 2024-11-12T17:34:30,590 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:32879/backupUT/backup_1731432850879/.backup.manifest 2024-11-12T17:34:30,592 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1731432850879 finished: type=FULL,tablelist=ns2:test-17314328406861;test-1731432840686,targetRootDir=hdfs://localhost:32879/backupUT,startts=1731432856602,completets=1731432870177,bytescopied=0 2024-11-12T17:34:30,592 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-12T17:34:30,593 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731432861312_ns2_test-17314328406861 2024-11-12T17:34:30,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1731432861312_ns2_test-17314328406861" type: DISABLED 2024-11-12T17:34:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731432861312_ns2_test-17314328406861 2024-11-12T17:34:30,601 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731432861312_ns2_test-17314328406861 for backup backup_1731432850879 succeeded. 2024-11-12T17:34:30,602 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731432857137_default_test-1731432840686 2024-11-12T17:34:30,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1731432857137_default_test-1731432840686" type: DISABLED 2024-11-12T17:34:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731432857137_default_test-1731432840686 2024-11-12T17:34:30,606 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731432857137_default_test-1731432840686 for backup backup_1731432850879 succeeded. 2024-11-12T17:34:30,607 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-12T17:34:30,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-12T17:34:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-12T17:34:30,620 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-12T17:34:30,626 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-12T17:34:30,640 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1731432850879 completed. 2024-11-12T17:34:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:34:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:34:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:backup_1731432850879', locateType=CURRENT is [region=backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd., hostname=1f32af3374c6,39493,1731432832063, seqNum=2] 2024-11-12T17:34:30,824 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(113): writing 99 rows to test-1731432840686 2024-11-12T17:34:30,845 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(116): written 99 rows to test-1731432840686 2024-11-12T17:34:30,998 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(120): written 111 rows to test-1731432840686 to Mob enabled CF 2024-11-12T17:34:31,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17314328406861', row='row-t20', locateType=CURRENT is [region=ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067., hostname=1f32af3374c6,39493,1731432832063, seqNum=2] 2024-11-12T17:34:31,026 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(133): written 5 rows to ns2:test-17314328406861 2024-11-12T17:34:31,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$3(2313): Client=jenkins//172.17.0.2 split test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:31,033 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] assignment.SplitTableRegionProcedure(210): test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. splitKey isn't explicitly specified, will try to find a best split key from RS 1f32af3374c6,39493,1731432832063 2024-11-12T17:34:31,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/13333c2c3c1a42179067d2c31f89d94b because midkey is the same as first or last row 2024-11-12T17:34:31,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f3/db6ea8024d2f42608b55815168d39424 because midkey is the same as first or last row 2024-11-12T17:34:31,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/5c0434f3a70a41059b32fac676fc1ec2 because midkey is the same as first or last row 2024-11-12T17:34:31,037 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.HRegion(2902): Flushing f14197f5e90bb7bd826d4a3ae95e1d7e 3/3 column families, dataSize=11.48 KB heapSize=27.79 KB 2024-11-12T17:34:31,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/f/fedf89305dee474fbf5f7539130b7d50 is 59, key is row-test-1731432840686-1-10/f:q1/1731432870682/Put/seqid=0 2024-11-12T17:34:31,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741880_1056 (size=10796) 2024-11-12T17:34:31,461 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/f/fedf89305dee474fbf5f7539130b7d50 2024-11-12T17:34:31,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112171447fa3da54c0aa83bc727d43d6045_f14197f5e90bb7bd826d4a3ae95e1d7e is 63, key is row-test-1731432840686-3-100/mob:q1/1731432870986/Put/seqid=0 2024-11-12T17:34:31,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741881_1057 (size=5591) 2024-11-12T17:34:31,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:31,895 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112171447fa3da54c0aa83bc727d43d6045_f14197f5e90bb7bd826d4a3ae95e1d7e to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/mobdir/data/default/test-1731432840686/af300c6268089b7a8d3f7acc33e67507/mob/d41d8cd98f00b204e9800998ecf8427e20241112171447fa3da54c0aa83bc727d43d6045_f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:31,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/mob/751cbd875328413f90578c10809be672, store: [table=test-1731432840686 family=mob region=f14197f5e90bb7bd826d4a3ae95e1d7e] 2024-11-12T17:34:31,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/mob/751cbd875328413f90578c10809be672 is 192, key is row-test-1731432840686-3-100/mob:q1/1731432870986/Put/seqid=0 2024-11-12T17:34:31,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741882_1058 (size=13760) 2024-11-12T17:34:32,302 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=6.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/mob/751cbd875328413f90578c10809be672 2024-11-12T17:34:32,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/f/fedf89305dee474fbf5f7539130b7d50 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/fedf89305dee474fbf5f7539130b7d50 2024-11-12T17:34:32,316 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/fedf89305dee474fbf5f7539130b7d50, entries=99, sequenceid=334, filesize=10.5 K 2024-11-12T17:34:32,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/mob/751cbd875328413f90578c10809be672 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/751cbd875328413f90578c10809be672 2024-11-12T17:34:32,324 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/751cbd875328413f90578c10809be672, entries=111, sequenceid=334, filesize=13.4 K 2024-11-12T17:34:32,325 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.48 KB/11754, heapSize ~27.51 KB/28168, currentSize=0 B/0 for f14197f5e90bb7bd826d4a3ae95e1d7e in 1288ms, sequenceid=334, compaction requested=false 2024-11-12T17:34:32,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.HRegion(2603): Flush status journal for f14197f5e90bb7bd826d4a3ae95e1d7e: 2024-11-12T17:34:32,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/fedf89305dee474fbf5f7539130b7d50 because midkey is the same as first or last row 2024-11-12T17:34:32,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f3/db6ea8024d2f42608b55815168d39424 because midkey is the same as first or last row 2024-11-12T17:34:32,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/751cbd875328413f90578c10809be672 because midkey is the same as first or last row 2024-11-12T17:34:32,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:32,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T17:34:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.CallRunner(138): callId: 302 service: MasterService methodName: SplitRegion size: 100 connection: 172.17.0.2:48310 deadline: 1731432931031, exception=org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row 2024-11-12T17:34:32,332 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: SPLIT_REGION, Table Name: default:test-1731432840686 failed with org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T17:34:32,332 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(144): region is not splittable, because java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T17:34:32,333 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T17:34:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1731432840686,, stopping at row=test-1731432840686 ,, for max=2147483647 with caching=100 2024-11-12T17:34:32,344 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1731432840686,, stopping at row=test-1731432840686 ,, for max=2147483647 with caching=100 2024-11-12T17:34:32,353 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(149): split finished in =1327 2024-11-12T17:34:32,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:34:32,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:34:32,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:34:32,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:34:32,393 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-12T17:34:32,397 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731432872353 2024-11-12T17:34:32,403 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1731432850879} 2024-11-12T17:34:32,404 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-12T17:34:32,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:34:32,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:34:32,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:34:32,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:34:32,449 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T17:34:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731432872449 (current time:1731432872449). 2024-11-12T17:34:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T17:34:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-12T17:34:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d39bdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:32,451 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:32,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:32,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:32,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@412012f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:32,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:32,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:32,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:32,453 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:32,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10fae378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:32,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:32,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:32,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:32,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475. 2024-11-12T17:34:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T17:34:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:32,457 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:34:32,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T17:34:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T17:34:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-12T17:34:32,460 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T17:34:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-12T17:34:32,461 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T17:34:32,464 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T17:34:32,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741883_1059 (size=52) 2024-11-12T17:34:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-12T17:34:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-12T17:34:32,870 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T17:34:32,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd}] 2024-11-12T17:34:32,872 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:33,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39493 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2024-11-12T17:34:33,024 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:34:33,024 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2902): Flushing 1a91481e8504b1563b1463c687c54efd 2/2 column families, dataSize=1.44 KB heapSize=2.75 KB 2024-11-12T17:34:33,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/meta/dae3103a2869480c805494f0d642f915 is 158, key is trslm:hdfs://localhost:32879/backupUT\x00ns2:test-17314328406861/meta:log-roll-map/1731432870165/Put/seqid=0 2024-11-12T17:34:33,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741884_1060 (size=5914) 2024-11-12T17:34:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-12T17:34:33,446 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/meta/dae3103a2869480c805494f0d642f915 2024-11-12T17:34:33,457 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/cd67ce9189984d3e9f4c1dee67666da9 is 433, key is session:backup_1731432850879/session:context/1731432870625/Put/seqid=0 2024-11-12T17:34:33,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741885_1061 (size=5535) 2024-11-12T17:34:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-12T17:34:33,862 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/cd67ce9189984d3e9f4c1dee67666da9 2024-11-12T17:34:33,870 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/meta/dae3103a2869480c805494f0d642f915 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/dae3103a2869480c805494f0d642f915 2024-11-12T17:34:33,876 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/dae3103a2869480c805494f0d642f915, entries=6, sequenceid=17, filesize=5.8 K 2024-11-12T17:34:33,877 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/cd67ce9189984d3e9f4c1dee67666da9 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cd67ce9189984d3e9f4c1dee67666da9 2024-11-12T17:34:33,883 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cd67ce9189984d3e9f4c1dee67666da9, entries=2, sequenceid=17, filesize=5.4 K 2024-11-12T17:34:33,884 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3140): Finished flush of dataSize ~1.44 KB/1478, heapSize ~2.72 KB/2784, currentSize=0 B/0 for 1a91481e8504b1563b1463c687c54efd in 860ms, sequenceid=17, compaction requested=false 2024-11-12T17:34:33,884 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2603): Flush status journal for 1a91481e8504b1563b1463c687c54efd: 2024-11-12T17:34:33,884 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. for snapshot_backup_system completed. 2024-11-12T17:34:33,884 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.' region-info for snapshot=snapshot_backup_system 2024-11-12T17:34:33,884 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T17:34:33,884 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/dae3103a2869480c805494f0d642f915] hfiles 2024-11-12T17:34:33,884 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/dae3103a2869480c805494f0d642f915 for snapshot=snapshot_backup_system 2024-11-12T17:34:33,884 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cd67ce9189984d3e9f4c1dee67666da9] hfiles 2024-11-12T17:34:33,884 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6 for snapshot=snapshot_backup_system 2024-11-12T17:34:33,884 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cd67ce9189984d3e9f4c1dee67666da9 for snapshot=snapshot_backup_system 2024-11-12T17:34:33,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741886_1062 (size=173) 2024-11-12T17:34:34,290 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:34:34,290 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-11-12T17:34:34,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-11-12T17:34:34,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:34,291 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:34,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-12T17:34:34,294 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T17:34:34,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd in 1.4220 sec 2024-11-12T17:34:34,295 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T17:34:34,296 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T17:34:34,296 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-12T17:34:34,297 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-12T17:34:34,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741887_1063 (size=810) 2024-11-12T17:34:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-12T17:34:34,707 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T17:34:34,716 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T17:34:34,717 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_backup_system 2024-11-12T17:34:34,718 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T17:34:34,718 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-12T17:34:34,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2600 sec 2024-11-12T17:34:35,431 DEBUG [master/1f32af3374c6:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1731432840686 2024-11-12T17:34:35,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741888_1064 (size=950) 2024-11-12T17:34:35,844 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/.tabledesc/.tableinfo.0000000002.908 2024-11-12T17:34:35,844 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/.tabledesc/.tableinfo.0000000003.950 2024-11-12T17:34:35,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1731432840686 in 417 msec 2024-11-12T17:34:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-12T17:34:36,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-12T17:34:36,622 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1731432872353 started at 1731432876622. 2024-11-12T17:34:36,624 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1731432872353 has been started. 2024-11-12T17:34:36,630 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [test-1731432840686, ns2:test-17314328406861] 2024-11-12T17:34:36,634 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1731432842004for backupID backup_1731432872353 2024-11-12T17:34:36,634 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-12T17:34:36,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-12T17:34:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-12T17:34:36,635 INFO [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-12T17:34:36,635 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-12T17:34:36,635 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-12T17:34:36,636 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-12T17:34:36,636 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T17:34:36,636 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T17:34:36,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T17:34:36,638 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,639 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T17:34:36,639 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T17:34:36,639 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,639 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-12T17:34:36,639 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-12T17:34:36,640 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T17:34:36,640 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-12T17:34:36,640 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T17:34:36,640 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-12T17:34:36,641 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-12T17:34:36,641 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-12T17:34:36,641 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-12T17:34:36,641 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-12T17:34:36,641 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-12T17:34:36,641 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-12T17:34:36,641 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '1f32af3374c6,39493,1731432832063' joining acquired barrier for procedure (rolllog) in zk 2024-11-12T17:34:36,643 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:36,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,643 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,643 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T17:34:36,643 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T17:34:36,643 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-12T17:34:36,643 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-12T17:34:36,643 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T17:34:36,644 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T17:34:36,644 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:36,645 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,645 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T17:34:36,646 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '1f32af3374c6,39493,1731432832063' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-12T17:34:36,646 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@2a1189ae[Count = 0] remaining members to acquire global barrier 2024-11-12T17:34:36,646 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-12T17:34:36,646 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:36,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:36,648 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:36,648 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:36,648 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-12T17:34:36,648 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,648 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-12T17:34:36,648 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-12T17:34:36,648 DEBUG [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,648 INFO [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1731432856655 highest: 1731432856714 on 1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,648 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0:(num 1731432856655) roll requested 2024-11-12T17:34:36,662 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432876649, exclude list is [], retry=0 2024-11-12T17:34:36,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:34:36,668 INFO [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432856655 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432876649 2024-11-12T17:34:36,668 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:34:36,668 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432856655 is not closed yet, will try archiving it next time 2024-11-12T17:34:36,668 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 1f32af3374c6%2C39493%2C1731432832063.meta:.meta(num 1731432856684) roll requested 2024-11-12T17:34:36,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741857_1033 (size=1198) 2024-11-12T17:34:36,670 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432856655 not finished, retry = 0 2024-11-12T17:34:36,684 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432876668.meta, exclude list is [], retry=0 2024-11-12T17:34:36,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:34:36,688 INFO [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432856684.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432876668.meta 2024-11-12T17:34:36,688 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:34:36,689 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432856684.meta is not closed yet, will try archiving it next time 2024-11-12T17:34:36,689 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1:(num 1731432856714) roll requested 2024-11-12T17:34:36,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741858_1034 (size=93) 2024-11-12T17:34:36,691 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432856684.meta to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs/1f32af3374c6%2C39493%2C1731432832063.meta.1731432856684.meta 2024-11-12T17:34:36,709 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432876689, exclude list is [], retry=0 2024-11-12T17:34:36,711 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:34:36,713 INFO [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432856714 with entries=224, filesize=29.77 KB; new WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432876689 2024-11-12T17:34:36,713 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:34:36,713 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432856714 is not closed yet, will try archiving it next time 2024-11-12T17:34:36,713 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432842004 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432842004 2024-11-12T17:34:36,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741859_1035 (size=30490) 2024-11-12T17:34:36,714 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432856714 not finished, retry = 0 2024-11-12T17:34:36,729 DEBUG [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 81 2024-11-12T17:34:36,729 INFO [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1731432876649 on 1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:34:36,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:34:36,747 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-12T17:34:36,747 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '1f32af3374c6,39493,1731432832063' in zk 2024-11-12T17:34:36,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,750 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-12T17:34:36,750 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T17:34:36,750 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-12T17:34:36,750 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,750 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T17:34:36,750 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T17:34:36,751 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T17:34:36,751 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T17:34:36,752 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:36,752 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,752 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T17:34:36,753 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:36,753 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,754 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '1f32af3374c6,39493,1731432832063': 2024-11-12T17:34:36,754 INFO [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-12T17:34:36,754 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '1f32af3374c6,39493,1731432832063' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-12T17:34:36,755 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-12T17:34:36,755 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-12T17:34:36,755 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-12T17:34:36,755 INFO [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-12T17:34:36,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T17:34:36,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T17:34:36,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T17:34:36,757 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-12T17:34:36,757 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T17:34:36,757 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-12T17:34:36,757 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T17:34:36,757 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T17:34:36,758 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T17:34:36,758 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T17:34:36,758 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T17:34:36,758 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,759 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T17:34:36,759 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:36,759 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T17:34:36,760 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:36,760 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,760 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,761 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T17:34:36,761 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:36,761 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T17:34:36,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T17:34:36,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:36,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T17:34:36,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:36,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:36,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T17:34:36,764 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T17:34:36,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T17:34:36,764 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T17:34:36,764 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T17:34:36,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-12T17:34:36,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-12T17:34:36,764 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2024-11-12T17:34:36,764 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T17:34:36,764 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T17:34:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-12T17:34:36,774 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {1f32af3374c6:39493=1731432842004} newestTimestamps: {1f32af3374c6:39493=1731432856714} 2024-11-12T17:34:36,777 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 1f32af3374c6%2C39493%2C1731432832063.meta.1731432835143.meta 2024-11-12T17:34:36,777 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 1f32af3374c6%2C39493%2C1731432832063.meta.1731432876668.meta 2024-11-12T17:34:36,777 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432856655 2024-11-12T17:34:36,777 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432876649 2024-11-12T17:34:36,777 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432856714 2024-11-12T17:34:36,777 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432876689 2024-11-12T17:34:36,778 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs/1f32af3374c6%2C39493%2C1731432832063.meta.1731432856684.meta 2024-11-12T17:34:36,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741892_1068 (size=364) 2024-11-12T17:34:37,189 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17314328406861 target: hdfs://localhost:32879/backupUT/backup_1731432872353/ns2/test-17314328406861 descriptor: 'ns2:test-17314328406861', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:34:37,189 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-12T17:34:37,192 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17314328406861 2024-11-12T17:34:37,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741893_1069 (size=50) 2024-11-12T17:34:37,599 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17314328406861 2024-11-12T17:34:37,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741894_1070 (size=950) 2024-11-12T17:34:38,011 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1731432840686 target: hdfs://localhost:32879/backupUT/backup_1731432872353/default/test-1731432840686 descriptor: 'test-1731432840686', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:34:38,011 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-12T17:34:38,014 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1731432840686 2024-11-12T17:34:38,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741895_1071 (size=53) 2024-11-12T17:34:38,420 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1731432840686 2024-11-12T17:34:38,430 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:32879/backupUT/.tmp/backup_1731432872353 from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432856655;hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432856714 2024-11-12T17:34:38,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be846bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:38,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:38,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:38,434 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:38,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:38,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:38,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@481db393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:38,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:38,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:38,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:38,435 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:38,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@254e7734, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:38,436 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T17:34:38,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 1f32af3374c6,39475,1731432830222 2024-11-12T17:34:38,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2ca11691 2024-11-12T17:34:38,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T17:34:38,438 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T17:34:38,444 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-12T17:34:38,444 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1731432840686 2024-11-12T17:34:38,445 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1731432840686,, stopping at row=test-1731432840686 ,, for max=2147483647 with caching=100 2024-11-12T17:34:38,445 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:38,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:38,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:38,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:38,449 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1731432840686: test-1731432840686; 2024-11-12T17:34:38,450 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17314328406861 2024-11-12T17:34:38,450 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17314328406861,, stopping at row=ns2:test-17314328406861 ,, for max=2147483647 with caching=100 2024-11-12T17:34:38,452 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17314328406861: ns2:test-17314328406861; 2024-11-12T17:34:38,452 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-12T17:34:38,453 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/hbase-staging/partitions_7a6c6a6b-7b4e-49c7-8b90-711aade8e031 2024-11-12T17:34:38,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741896_1072 (size=192) 2024-11-12T17:34:38,877 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:38,878 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:38,878 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,869 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop-7426690119958801949.jar 2024-11-12T17:34:39,870 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,870 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,870 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,871 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,871 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,871 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,871 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,872 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,872 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,872 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T17:34:39,872 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T17:34:39,873 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T17:34:39,873 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T17:34:39,873 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T17:34:39,873 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T17:34:39,874 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T17:34:39,874 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T17:34:39,874 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T17:34:39,874 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T17:34:39,875 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T17:34:39,876 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,876 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,877 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,877 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,877 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,878 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:34:39,878 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:34:39,878 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: default:test-1731432840686,ns2:test-17314328406861 2024-11-12T17:34:39,878 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T17:34:39,878 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:154) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T17:34:39,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:39,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:39,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:34:39,879 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T17:34:40,865 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop-5294350257096726106.jar 2024-11-12T17:34:40,879 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T17:34:40,906 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432856655 for WAL files 2024-11-12T17:34:40,907 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432856655 2024-11-12T17:34:40,908 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432856714 for WAL files 2024-11-12T17:34:40,909 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432856714 2024-11-12T17:34:41,010 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local1648103600_0003_e1875e9a-58ca-4fc5-a0cd-7b766a639a60/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-12T17:34:41,010 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local1648103600_0003_e1875e9a-58ca-4fc5-a0cd-7b766a639a60/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-12T17:34:41,100 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432856714 for hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432856714 (-9223372036854775808:9223372036854775807) length:30490 2024-11-12T17:34:41,118 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-12T17:34:41,118 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-12T17:34:41,167 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432856655 for hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432856655 (-9223372036854775808:9223372036854775807) length:1198 2024-11-12T17:34:41,171 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-12T17:34:41,171 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-12T17:34:41,255 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@272bb456, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:41,255 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:41,255 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:41,256 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:41,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:41,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:41,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e62c04d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:41,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:41,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:41,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:41,258 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:41,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5793e21c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:41,259 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:41,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:41,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:41,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:41,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17314328406861', row='row-t20', locateType=CURRENT is [region=ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067., hostname=1f32af3374c6,39493,1731432832063, seqNum=2] 2024-11-12T17:34:41,264 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-12T17:34:41,264 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T17:34:41,265 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:41,265 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:41,265 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:34:41,265 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-11-12T17:34:41,265 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 1f32af3374c6 2024-11-12T17:34:41,271 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/backupUT/.tmp/backup_1731432872353/_temporary/0/_temporary/attempt_local1648103600_0003_r_000000_0/ns2/test-17314328406861/f/f9b51b9ef28249abbf61b1603ca6a131 is 38, key is row-t20/f:q1/1731432871014/Put/seqid=0 2024-11-12T17:34:41,273 WARN [Thread-950 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [1f32af3374c6:39493] Specified favored nodes: [1f32af3374c6:39493] 2024-11-12T17:34:41,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741897_1073 (size=5275) 2024-11-12T17:34:41,704 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d0b997e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:41,704 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:41,705 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:41,706 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:41,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:41,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:41,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eba5693, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:41,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:41,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:41,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:41,707 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:41,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ba38ac5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:41,709 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:41,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:41,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:41,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:41,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1731432840686', row='row-test-1731432840686-1-0', locateType=CURRENT is [region=test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e., hostname=1f32af3374c6,39493,1731432832063, seqNum=107] 2024-11-12T17:34:41,713 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-12T17:34:41,713 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T17:34:41,713 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:41,713 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:41,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:34:41,713 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1731432840686-1-0] 2024-11-12T17:34:41,713 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 1f32af3374c6 2024-11-12T17:34:41,724 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5114c9ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:41,724 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:41,724 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:41,725 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:41,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:41,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:41,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe42847, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:41,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:41,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:41,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:41,726 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:41,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74134b6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:41,727 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:41,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:41,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:41,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:41,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1731432840686', row='row-test-1731432840686-3-0', locateType=CURRENT is [region=test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e., hostname=1f32af3374c6,39493,1731432832063, seqNum=107] 2024-11-12T17:34:41,731 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-12T17:34:41,731 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T17:34:41,731 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:41,731 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:41,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:34:41,731 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1731432840686-3-0] 2024-11-12T17:34:41,731 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 1f32af3374c6 2024-11-12T17:34:41,740 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/backupUT/.tmp/backup_1731432872353/_temporary/0/_temporary/attempt_local1648103600_0003_r_000001_0/default/test-1731432840686/f/0fcd9b73be514a85b7992745d103ab02 is 59, key is row-test-1731432840686-1-10/f:q1/1731432870682/Put/seqid=0 2024-11-12T17:34:41,742 WARN [Thread-962 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [1f32af3374c6:39493] Specified favored nodes: [1f32af3374c6:39493] 2024-11-12T17:34:41,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741898_1074 (size=10770) 2024-11-12T17:34:42,146 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/backupUT/.tmp/backup_1731432872353/_temporary/0/_temporary/attempt_local1648103600_0003_r_000001_0/default/test-1731432840686/mob/6ee68a740a0144cfaf29f1a0c9eced28 is 63, key is row-test-1731432840686-3-100/mob:q1/1731432870986/Put/seqid=0 2024-11-12T17:34:42,148 WARN [Thread-964 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [1f32af3374c6:39493] Specified favored nodes: [1f32af3374c6:39493] 2024-11-12T17:34:42,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741899_1075 (size=11811) 2024-11-12T17:34:43,055 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:32879/backupUT 2024-11-12T17:34:43,055 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1731432872353 2024-11-12T17:34:43,056 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-11-12T17:34:43,077 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:32879/backupUT/.tmp/backup_1731432872353, hdfs://localhost:32879/backupUT] 2024-11-12T17:34:43,150 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T17:34:43,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741900_1076 (size=5275) 2024-11-12T17:34:43,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741901_1077 (size=10770) 2024-11-12T17:34:44,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741902_1078 (size=11811) 2024-11-12T17:34:44,764 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-11-12T17:34:44,767 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1731432872353 2024-11-12T17:34:44,767 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 27856 bytes copied." 2024-11-12T17:34:44,768 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local931392828_0004 completed: true true 2024-11-12T17:34:44,769 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=3823807 FILE: Number of bytes written=6189849 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=231467 HDFS: Number of bytes written=2433185 HDFS: Number of read operations=643 HDFS: Number of large read operations=0 HDFS: Number of write operations=249 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=12 Map output records=0 Input split bytes=339 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=3004 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=27856 Bytes Copied=27856 Bytes Expected=27856 Files Copied=4 DIR_COPY=8 2024-11-12T17:34:44,769 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:32879/backupUT/.tmp/backup_1731432872353 to hdfs://localhost:32879/backupUT finished. 2024-11-12T17:34:44,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system_bulk', row='bulk:ns2:test-17314328406861:', locateType=CURRENT is [region=backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c., hostname=1f32af3374c6,39493,1731432832063, seqNum=2] 2024-11-12T17:34:44,784 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731432872353 2024-11-12T17:34:44,787 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1731432850879} 2024-11-12T17:34:44,787 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-12T17:34:44,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741903_1079 (size=384) 2024-11-12T17:34:45,193 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:32879/backupUT/backup_1731432872353/.backup.manifest 2024-11-12T17:34:45,193 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1731432872353 finished: type=INCREMENTAL,tablelist=ns2:test-17314328406861;test-1731432840686,targetRootDir=hdfs://localhost:32879/backupUT,startts=1731432876622,completets=1731432884784,bytescopied=0 2024-11-12T17:34:45,193 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-12T17:34:45,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-12T17:34:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-12T17:34:45,202 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-12T17:34:45,204 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-12T17:34:45,211 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1731432872353 completed. 2024-11-12T17:34:45,211 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-11-12T17:34:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:34:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:34:45,228 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:32879/backupUT/backup_1731432872353/.backup.manifest 2024-11-12T17:34:45,231 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1731432872353/.backup.manifest 2024-11-12T17:34:45,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1731432840686 from 'test-1731432840686', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1731432840686', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:34:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1731432840686 2024-11-12T17:34:45,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741904_1080 (size=908) 2024-11-12T17:34:45,246 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/.tabledesc/.tableinfo.0000000003.950 2024-11-12T17:34:45,246 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/.tabledesc/.tableinfo.0000000004.908 2024-11-12T17:34:45,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1731432840686}] 2024-11-12T17:34:45,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, REOPEN/MOVE}] 2024-11-12T17:34:45,253 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, REOPEN/MOVE 2024-11-12T17:34:45,254 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=f14197f5e90bb7bd826d4a3ae95e1d7e, regionState=CLOSING, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:45,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, REOPEN/MOVE because future has completed 2024-11-12T17:34:45,256 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T17:34:45,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e, server=1f32af3374c6,39493,1731432832063}] 2024-11-12T17:34:45,409 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(122): Close f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:45,409 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T17:34:45,410 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1722): Closing f14197f5e90bb7bd826d4a3ae95e1d7e, disabling compactions & flushes 2024-11-12T17:34:45,410 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1755): Closing region test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:45,410 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1776): Time limited wait for close lock on test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:45,410 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1843): Acquired close lock on test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. after waiting 0 ms 2024-11-12T17:34:45,410 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1853): Updates disabled for region test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:45,415 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=106 2024-11-12T17:34:45,416 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T17:34:45,416 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1973): Closed test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:45,416 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1676): Region close journal for f14197f5e90bb7bd826d4a3ae95e1d7e: Waiting for close lock at 1731432885409Running coprocessor pre-close hooks at 1731432885409Disabling compacts and flushes for region at 1731432885409Disabling writes for close at 1731432885410 (+1 ms)Writing region close event to WAL at 1731432885411 (+1 ms)Running coprocessor post-close hooks at 1731432885416 (+5 ms)Closed at 1731432885416 2024-11-12T17:34:45,416 WARN [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegionServer(3299): Not adding moved region record: f14197f5e90bb7bd826d4a3ae95e1d7e to self. 2024-11-12T17:34:45,418 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(157): Closed f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:45,419 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=f14197f5e90bb7bd826d4a3ae95e1d7e, regionState=CLOSED 2024-11-12T17:34:45,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e, server=1f32af3374c6,39493,1731432832063 because future has completed 2024-11-12T17:34:45,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-12T17:34:45,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e, server=1f32af3374c6,39493,1731432832063 in 165 msec 2024-11-12T17:34:45,424 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, REOPEN/MOVE; state=CLOSED, location=1f32af3374c6,39493,1731432832063; forceNewPlan=false, retain=true 2024-11-12T17:34:45,428 DEBUG [master/1f32af3374c6:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1731432840686 2024-11-12T17:34:45,428 DEBUG [master/1f32af3374c6:0.Chore.1 {}] procedure.MasterProcedureScheduler(153): There is already a procedure running for table test-1731432840686, added pid=46, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1731432840686 to waiting queue 2024-11-12T17:34:45,575 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=f14197f5e90bb7bd826d4a3ae95e1d7e, regionState=OPENING, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:45,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, REOPEN/MOVE because future has completed 2024-11-12T17:34:45,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e, server=1f32af3374c6,39493,1731432832063}] 2024-11-12T17:34:45,733 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:45,733 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => f14197f5e90bb7bd826d4a3ae95e1d7e, NAME => 'test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e.', STARTKEY => '', ENDKEY => ''} 2024-11-12T17:34:45,734 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T17:34:45,734 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1731432840686 f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:45,734 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:34:45,734 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:45,734 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:45,736 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:45,737 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f14197f5e90bb7bd826d4a3ae95e1d7e columnFamilyName f 2024-11-12T17:34:45,737 DEBUG [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:45,744 DEBUG [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/13333c2c3c1a42179067d2c31f89d94b 2024-11-12T17:34:45,748 DEBUG [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/fedf89305dee474fbf5f7539130b7d50 2024-11-12T17:34:45,749 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.HStore(327): Store=f14197f5e90bb7bd826d4a3ae95e1d7e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:34:45,749 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:45,750 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f14197f5e90bb7bd826d4a3ae95e1d7e columnFamilyName f2 2024-11-12T17:34:45,750 DEBUG [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:45,751 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.HStore(327): Store=f14197f5e90bb7bd826d4a3ae95e1d7e/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:34:45,751 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:45,752 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f14197f5e90bb7bd826d4a3ae95e1d7e columnFamilyName mob 2024-11-12T17:34:45,752 DEBUG [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:45,759 DEBUG [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/5c0434f3a70a41059b32fac676fc1ec2 2024-11-12T17:34:45,763 DEBUG [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/751cbd875328413f90578c10809be672 2024-11-12T17:34:45,763 INFO [StoreOpener-f14197f5e90bb7bd826d4a3ae95e1d7e-1 {}] regionserver.HStore(327): Store=f14197f5e90bb7bd826d4a3ae95e1d7e/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:34:45,764 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:45,765 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:45,766 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:45,767 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:45,767 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:45,767 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1731432840686 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T17:34:45,769 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:45,770 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened f14197f5e90bb7bd826d4a3ae95e1d7e; next sequenceid=338; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9813824320, jitterRate=-0.08601638674736023}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T17:34:45,770 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:45,772 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for f14197f5e90bb7bd826d4a3ae95e1d7e: Running coprocessor pre-open hook at 1731432885734Writing region info on filesystem at 1731432885734Initializing all the Stores at 1731432885735 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432885735Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432885736 (+1 ms)Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432885736Cleaning up temporary data from old regions at 1731432885767 (+31 ms)Running coprocessor post-open hooks at 1731432885770 (+3 ms)Region opened successfully at 1731432885772 (+2 ms) 2024-11-12T17:34:45,773 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e., pid=47, masterSystemTime=1731432885729 2024-11-12T17:34:45,775 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:45,775 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:45,775 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=f14197f5e90bb7bd826d4a3ae95e1d7e, regionState=OPEN, openSeqNum=338, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:34:45,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e, server=1f32af3374c6,39493,1731432832063 because future has completed 2024-11-12T17:34:45,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=44 2024-11-12T17:34:45,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=44, state=SUCCESS, hasLock=false; OpenRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e, server=1f32af3374c6,39493,1731432832063 in 201 msec 2024-11-12T17:34:45,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-12T17:34:45,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1731432840686, region=f14197f5e90bb7bd826d4a3ae95e1d7e, REOPEN/MOVE in 529 msec 2024-11-12T17:34:45,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-12T17:34:45,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1731432840686 in 533 msec 2024-11-12T17:34:45,786 DEBUG [PEWorker-1 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=test-1731432840686 2024-11-12T17:34:45,792 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f3/db6ea8024d2f42608b55815168d39424 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/archive/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f3/db6ea8024d2f42608b55815168d39424 2024-11-12T17:34:45,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1731432840686 in 561 msec 2024-11-12T17:34:45,795 DEBUG [PEWorker-1 {}] procedure.MasterProcedureScheduler(373): pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1731432840686 completed, enqueue a new procedure pid=46, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1731432840686 2024-11-12T17:34:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T17:34:45,796 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1731432840686 completed 2024-11-12T17:34:45,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741905_1081 (size=950) 2024-11-12T17:34:45,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:34:45,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:34:45,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:34:45,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:34:45,831 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-12T17:34:45,835 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731432885797 2024-11-12T17:34:45,838 DEBUG [Time-limited test {}] impl.TableBackupClient(330): Dependent incremental backup image: {BackupID=backup_1731432872353} 2024-11-12T17:34:45,838 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1731432850879} 2024-11-12T17:34:45,838 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 2 ancestors for the current backup. 2024-11-12T17:34:45,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:34:45,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:34:45,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:34:45,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:34:45,878 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected Exception : Mismatch in column family descriptor for table: test-1731432840686 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException: Mismatch in column family descriptor for table: test-1731432840686 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException$ColumnFamilyMismatchExceptionBuilder.build(ColumnFamilyMismatchException.java:56) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.verifyCfCompatibility(IncrementalTableBackupClient.java:513) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:280) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.lambda$TestIncBackupRestore$0(TestIncrementalBackup.java:171) ~[test-classes/:?] at org.junit.Assert.assertThrows(Assert.java:1001) ~[junit-4.13.2.jar:4.13.2] at org.junit.Assert.assertThrows(Assert.java:981) ~[junit-4.13.2.jar:4.13.2] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:170) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T17:34:45,881 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1731432885797,startts=0,failedts=1731432885879,failedphase=REQUEST,failedmessage= Mismatch in column family descriptor for table: test-1731432840686 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'tru 2024-11-12T17:34:45,881 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-12T17:34:45,882 WARN [Time-limited test {}] impl.BackupSystemTable(1549): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2024-11-12T17:34:45,883 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-12T17:34:45,884 ERROR [Time-limited test {}] impl.BackupSystemTable(1578): Snapshot snapshot_backup_system does not exists 2024-11-12T17:34:45,884 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: REQUEST 2024-11-12T17:34:45,886 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-12T17:34:45,888 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1731432885797 failed. 2024-11-12T17:34:45,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:34:45,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:34:45,905 INFO [Time-limited test {}] impl.BackupManager(225): Full backup all the tables available in the cluster: [backup:system_bulk, test-1731432840686, ns2:test-17314328406861, ns3:test-17314328406862, ns4:test-17314328406863] 2024-11-12T17:34:45,905 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-12T17:34:45,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T17:34:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731432885909 (current time:1731432885909). 2024-11-12T17:34:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T17:34:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-12T17:34:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d04b216, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:45,910 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:45,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:45,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:45,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54fcd0e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:45,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:45,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:45,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:45,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:45,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23c5f617, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:45,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:45,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:45,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:45,915 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475. 2024-11-12T17:34:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T17:34:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:45,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:34:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T17:34:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T17:34:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 48 2024-11-12T17:34:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T17:34:45,918 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T17:34:45,919 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T17:34:45,921 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T17:34:45,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741906_1082 (size=52) 2024-11-12T17:34:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T17:34:46,206 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/.tabledesc/.tableinfo.0000000004.908 2024-11-12T17:34:46,206 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/.tabledesc/.tableinfo.0000000005.950 2024-11-12T17:34:46,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1731432840686 in 779 msec 2024-11-12T17:34:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T17:34:46,328 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T17:34:46,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd}] 2024-11-12T17:34:46,329 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=49, ppid=48, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:46,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39493 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=49 2024-11-12T17:34:46,482 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:34:46,482 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 1a91481e8504b1563b1463c687c54efd 2/2 column families, dataSize=2.61 KB heapSize=3.99 KB 2024-11-12T17:34:46,487 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/meta/6f005e3ac6844baa93a9096523aa1ee7 is 158, key is trslm:hdfs://localhost:32879/backupUT\x00ns2:test-17314328406861/meta:log-roll-map/1731432884775/Put/seqid=0 2024-11-12T17:34:46,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741907_1083 (size=5745) 2024-11-12T17:34:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T17:34:46,892 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=492 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/meta/6f005e3ac6844baa93a9096523aa1ee7 2024-11-12T17:34:46,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/c625085d4bf443bcad8f22e2942797d3 is 1245, key is session:backup_1731432885797/session:context/1731432885885/Put/seqid=0 2024-11-12T17:34:46,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741908_1084 (size=6682) 2024-11-12T17:34:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T17:34:47,306 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/c625085d4bf443bcad8f22e2942797d3 2024-11-12T17:34:47,314 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/meta/6f005e3ac6844baa93a9096523aa1ee7 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/6f005e3ac6844baa93a9096523aa1ee7 2024-11-12T17:34:47,320 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/6f005e3ac6844baa93a9096523aa1ee7, entries=4, sequenceid=31, filesize=5.6 K 2024-11-12T17:34:47,321 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/c625085d4bf443bcad8f22e2942797d3 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/c625085d4bf443bcad8f22e2942797d3 2024-11-12T17:34:47,327 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/c625085d4bf443bcad8f22e2942797d3, entries=3, sequenceid=31, filesize=6.5 K 2024-11-12T17:34:47,328 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~2.61 KB/2669, heapSize ~3.96 KB/4056, currentSize=0 B/0 for 1a91481e8504b1563b1463c687c54efd in 846ms, sequenceid=31, compaction requested=true 2024-11-12T17:34:47,328 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 1a91481e8504b1563b1463c687c54efd: 2024-11-12T17:34:47,328 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. for snapshot_backup_system completed. 2024-11-12T17:34:47,328 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.' region-info for snapshot=snapshot_backup_system 2024-11-12T17:34:47,329 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T17:34:47,329 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/dae3103a2869480c805494f0d642f915, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/6f005e3ac6844baa93a9096523aa1ee7] hfiles 2024-11-12T17:34:47,329 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/dae3103a2869480c805494f0d642f915 for snapshot=snapshot_backup_system 2024-11-12T17:34:47,329 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/6f005e3ac6844baa93a9096523aa1ee7 for snapshot=snapshot_backup_system 2024-11-12T17:34:47,329 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cd67ce9189984d3e9f4c1dee67666da9, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/c625085d4bf443bcad8f22e2942797d3] hfiles 2024-11-12T17:34:47,329 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6 for snapshot=snapshot_backup_system 2024-11-12T17:34:47,329 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cd67ce9189984d3e9f4c1dee67666da9 for snapshot=snapshot_backup_system 2024-11-12T17:34:47,329 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/c625085d4bf443bcad8f22e2942797d3 for snapshot=snapshot_backup_system 2024-11-12T17:34:47,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741909_1085 (size=251) 2024-11-12T17:34:47,735 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:34:47,735 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-12T17:34:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-12T17:34:47,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:47,736 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=49, ppid=48, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:34:47,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-12T17:34:47,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd in 1.4090 sec 2024-11-12T17:34:47,739 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T17:34:47,740 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T17:34:47,740 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T17:34:47,740 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-12T17:34:47,741 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-12T17:34:47,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741910_1086 (size=888) 2024-11-12T17:34:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T17:34:48,152 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T17:34:48,160 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T17:34:48,161 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_backup_system 2024-11-12T17:34:48,162 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T17:34:48,162 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 48 2024-11-12T17:34:48,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2460 sec 2024-11-12T17:34:48,690 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 6dd38527fc6c5b38bfed421a5b5be067, had cached 0 bytes from a total of 8413 2024-11-12T17:34:49,070 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T17:34:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T17:34:50,082 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-12T17:34:50,082 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1731432885889 started at 1731432890082. 2024-11-12T17:34:50,084 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1731432885889 has been started. 2024-11-12T17:34:50,088 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-12T17:34:50,089 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-12T17:34:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-12T17:34:50,089 INFO [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-12T17:34:50,089 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-12T17:34:50,089 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-12T17:34:50,089 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-12T17:34:50,090 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T17:34:50,090 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T17:34:50,092 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T17:34:50,092 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T17:34:50,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T17:34:50,093 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,093 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-12T17:34:50,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-12T17:34:50,094 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T17:34:50,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-12T17:34:50,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T17:34:50,094 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-12T17:34:50,095 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-12T17:34:50,095 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-12T17:34:50,095 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-12T17:34:50,095 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-12T17:34:50,095 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-12T17:34:50,095 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-12T17:34:50,095 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '1f32af3374c6,39493,1731432832063' joining acquired barrier for procedure (rolllog) in zk 2024-11-12T17:34:50,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,097 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:50,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T17:34:50,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T17:34:50,098 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-12T17:34:50,098 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-12T17:34:50,098 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T17:34:50,098 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T17:34:50,099 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:50,099 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,099 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T17:34:50,100 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '1f32af3374c6,39493,1731432832063' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-12T17:34:50,100 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@48eb89a5[Count = 0] remaining members to acquire global barrier 2024-11-12T17:34:50,100 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-12T17:34:50,100 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:50,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:50,101 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:50,101 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:50,102 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-12T17:34:50,102 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,102 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-12T17:34:50,102 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-12T17:34:50,102 DEBUG [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,102 INFO [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1731432876649 highest: 1731432876689 on 1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,102 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0:(num 1731432876649) roll requested 2024-11-12T17:34:50,116 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432890102, exclude list is [], retry=0 2024-11-12T17:34:50,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:34:50,120 INFO [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432876649 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432890102 2024-11-12T17:34:50,120 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:34:50,120 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432876649 is not closed yet, will try archiving it next time 2024-11-12T17:34:50,120 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 1f32af3374c6%2C39493%2C1731432832063.meta:.meta(num 1731432876668) roll requested 2024-11-12T17:34:50,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741889_1065 (size=93) 2024-11-12T17:34:50,122 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432876649 not finished, retry = 0 2024-11-12T17:34:50,134 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432890121.meta, exclude list is [], retry=0 2024-11-12T17:34:50,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:34:50,137 INFO [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432876668.meta with entries=4, filesize=1.85 KB; new WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432890121.meta 2024-11-12T17:34:50,137 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:34:50,137 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432876668.meta is not closed yet, will try archiving it next time 2024-11-12T17:34:50,137 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1:(num 1731432876689) roll requested 2024-11-12T17:34:50,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741890_1066 (size=1903) 2024-11-12T17:34:50,139 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432876668.meta not finished, retry = 0 2024-11-12T17:34:50,150 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432890137, exclude list is [], retry=0 2024-11-12T17:34:50,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:34:50,153 INFO [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432876689 with entries=15, filesize=4.80 KB; new WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432890137 2024-11-12T17:34:50,154 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:34:50,154 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432876689 is not closed yet, will try archiving it next time 2024-11-12T17:34:50,154 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432856714 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432856714 2024-11-12T17:34:50,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741891_1067 (size=4927) 2024-11-12T17:34:50,155 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432876689 not finished, retry = 0 2024-11-12T17:34:50,162 DEBUG [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-12T17:34:50,162 INFO [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1731432890102 on 1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:34:50,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:34:50,178 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-12T17:34:50,179 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '1f32af3374c6,39493,1731432832063' in zk 2024-11-12T17:34:50,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,181 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-12T17:34:50,181 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T17:34:50,181 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-12T17:34:50,181 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,181 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T17:34:50,181 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T17:34:50,181 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T17:34:50,182 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T17:34:50,182 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:50,182 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,183 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T17:34:50,183 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:50,184 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,184 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '1f32af3374c6,39493,1731432832063': 2024-11-12T17:34:50,184 INFO [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-12T17:34:50,184 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '1f32af3374c6,39493,1731432832063' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-12T17:34:50,184 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-12T17:34:50,184 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-12T17:34:50,185 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-12T17:34:50,185 INFO [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-12T17:34:50,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T17:34:50,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T17:34:50,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T17:34:50,187 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-12T17:34:50,187 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-12T17:34:50,187 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T17:34:50,187 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T17:34:50,187 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T17:34:50,188 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T17:34:50,188 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T17:34:50,188 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T17:34:50,188 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T17:34:50,188 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,188 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:50,189 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T17:34:50,189 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:50,190 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,190 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,190 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T17:34:50,191 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:34:50,191 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T17:34:50,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T17:34:50,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T17:34:50,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:34:50,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:50,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T17:34:50,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T17:34:50,193 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T17:34:50,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T17:34:50,193 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T17:34:50,194 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T17:34:50,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-12T17:34:50,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-12T17:34:50,194 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T17:34:50,194 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T17:34:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-12T17:34:50,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1731432890203_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-12T17:34:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731432890204 (current time:1731432890204). 2024-11-12T17:34:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T17:34:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731432890203_backup_system_bulk VERSION not specified, setting to 2 2024-11-12T17:34:50,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2ee0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:50,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:50,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:50,206 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:50,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:50,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:50,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bfe9ec9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:50,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:50,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:50,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:50,207 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:50,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@322d0744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:50,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:50,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:50,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:50,210 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:50,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475. 2024-11-12T17:34:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T17:34:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:50,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:34:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T17:34:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731432890203_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-12T17:34:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731432890203_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-12T17:34:50,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T17:34:50,214 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731432890203_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T17:34:50,215 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731432890203_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T17:34:50,217 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731432890203_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T17:34:50,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741914_1090 (size=76) 2024-11-12T17:34:50,223 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432876649 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432876649 2024-11-12T17:34:50,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T17:34:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T17:34:50,624 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731432890203_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T17:34:50,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 569aa6aeafd4b9d079658f569537f60c}] 2024-11-12T17:34:50,625 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:34:50,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39493 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2024-11-12T17:34:50,777 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. 2024-11-12T17:34:50,777 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 569aa6aeafd4b9d079658f569537f60c: 2024-11-12T17:34:50,778 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. for snapshot_1731432890203_backup_system_bulk completed. 2024-11-12T17:34:50,778 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(241): Storing 'backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c.' region-info for snapshot=snapshot_1731432890203_backup_system_bulk 2024-11-12T17:34:50,778 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T17:34:50,778 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-12T17:34:50,778 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-12T17:34:50,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741915_1091 (size=61) 2024-11-12T17:34:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T17:34:50,944 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-12T17:34:51,184 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. 2024-11-12T17:34:51,184 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-12T17:34:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-12T17:34:51,185 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731432890203_backup_system_bulk on region 569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:34:51,185 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:34:51,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-12T17:34:51,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 569aa6aeafd4b9d079658f569537f60c in 562 msec 2024-11-12T17:34:51,187 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731432890203_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T17:34:51,188 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731432890203_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T17:34:51,189 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731432890203_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T17:34:51,189 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731432890203_backup_system_bulk 2024-11-12T17:34:51,190 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_1731432890203_backup_system_bulk 2024-11-12T17:34:51,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741916_1092 (size=702) 2024-11-12T17:34:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T17:34:51,600 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731432890203_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T17:34:51,605 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731432890203_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T17:34:51,606 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_1731432890203_backup_system_bulk to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_1731432890203_backup_system_bulk 2024-11-12T17:34:51,607 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731432890203_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T17:34:51,607 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731432890203_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-12T17:34:51,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731432890203_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } in 1.3950 sec 2024-11-12T17:34:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T17:34:52,361 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system_bulk completed 2024-11-12T17:34:52,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1731432892361_ns3_test-17314328406862 table=ns3:test-17314328406862 type=FLUSH ttl=0 } 2024-11-12T17:34:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731432892362 (current time:1731432892362). 2024-11-12T17:34:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T17:34:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731432892361_ns3_test-17314328406862 VERSION not specified, setting to 2 2024-11-12T17:34:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b06ea22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:52,363 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:52,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:52,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:52,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15e39b17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:52,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:52,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:52,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:52,365 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:52,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@723b6edf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:52,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:52,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:52,368 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:52,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475. 2024-11-12T17:34:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T17:34:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:52,369 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:34:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T17:34:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731432892361_ns3_test-17314328406862 table=ns3:test-17314328406862 type=FLUSH ttl=0 } 2024-11-12T17:34:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731432892361_ns3_test-17314328406862 table=ns3:test-17314328406862 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-12T17:34:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T17:34:52,372 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731432892361_ns3_test-17314328406862 table=ns3:test-17314328406862 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T17:34:52,373 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731432892361_ns3_test-17314328406862 table=ns3:test-17314328406862 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T17:34:52,375 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731432892361_ns3_test-17314328406862 table=ns3:test-17314328406862 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T17:34:52,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741917_1093 (size=86) 2024-11-12T17:34:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T17:34:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T17:34:52,781 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731432892361_ns3_test-17314328406862 table=ns3:test-17314328406862 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T17:34:52,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure ad7fb5b314f3c7778103a7cc5221da50}] 2024-11-12T17:34:52,782 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure ad7fb5b314f3c7778103a7cc5221da50 2024-11-12T17:34:52,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39493 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2024-11-12T17:34:52,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. 2024-11-12T17:34:52,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for ad7fb5b314f3c7778103a7cc5221da50: 2024-11-12T17:34:52,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. for snapshot_1731432892361_ns3_test-17314328406862 completed. 2024-11-12T17:34:52,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(241): Storing 'ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50.' region-info for snapshot=snapshot_1731432892361_ns3_test-17314328406862 2024-11-12T17:34:52,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T17:34:52,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-12T17:34:52,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741918_1094 (size=52) 2024-11-12T17:34:53,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T17:34:53,341 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. 2024-11-12T17:34:53,341 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-12T17:34:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-12T17:34:53,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731432892361_ns3_test-17314328406862 on region ad7fb5b314f3c7778103a7cc5221da50 2024-11-12T17:34:53,342 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure ad7fb5b314f3c7778103a7cc5221da50 2024-11-12T17:34:53,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-12T17:34:53,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; SnapshotRegionProcedure ad7fb5b314f3c7778103a7cc5221da50 in 562 msec 2024-11-12T17:34:53,345 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731432892361_ns3_test-17314328406862 table=ns3:test-17314328406862 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T17:34:53,346 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731432892361_ns3_test-17314328406862 table=ns3:test-17314328406862 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T17:34:53,347 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731432892361_ns3_test-17314328406862 table=ns3:test-17314328406862 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T17:34:53,347 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731432892361_ns3_test-17314328406862 2024-11-12T17:34:53,347 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_1731432892361_ns3_test-17314328406862 2024-11-12T17:34:53,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741919_1095 (size=443) 2024-11-12T17:34:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T17:34:53,759 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731432892361_ns3_test-17314328406862 table=ns3:test-17314328406862 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T17:34:53,766 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731432892361_ns3_test-17314328406862 table=ns3:test-17314328406862 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T17:34:53,766 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_1731432892361_ns3_test-17314328406862 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_1731432892361_ns3_test-17314328406862 2024-11-12T17:34:53,768 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731432892361_ns3_test-17314328406862 table=ns3:test-17314328406862 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T17:34:53,768 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731432892361_ns3_test-17314328406862 table=ns3:test-17314328406862 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-12T17:34:53,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731432892361_ns3_test-17314328406862 table=ns3:test-17314328406862 type=FLUSH ttl=0 } in 1.3980 sec 2024-11-12T17:34:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T17:34:54,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns3:test-17314328406862 completed 2024-11-12T17:34:54,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1731432894522_ns4_test-17314328406863 table=ns4:test-17314328406863 type=FLUSH ttl=0 } 2024-11-12T17:34:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731432894523 (current time:1731432894523). 2024-11-12T17:34:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T17:34:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731432894522_ns4_test-17314328406863 VERSION not specified, setting to 2 2024-11-12T17:34:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3219afa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:54,525 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:54,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:54,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:54,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64028220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:54,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:54,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:54,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:54,526 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:54,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aae2389, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:54,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:54,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:54,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:54,530 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475. 2024-11-12T17:34:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T17:34:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:54,531 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:34:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T17:34:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731432894522_ns4_test-17314328406863 table=ns4:test-17314328406863 type=FLUSH ttl=0 } 2024-11-12T17:34:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731432894522_ns4_test-17314328406863 table=ns4:test-17314328406863 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-12T17:34:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T17:34:54,533 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731432894522_ns4_test-17314328406863 table=ns4:test-17314328406863 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T17:34:54,534 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731432894522_ns4_test-17314328406863 table=ns4:test-17314328406863 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T17:34:54,536 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731432894522_ns4_test-17314328406863 table=ns4:test-17314328406863 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T17:34:54,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741920_1096 (size=86) 2024-11-12T17:34:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T17:34:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T17:34:54,943 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731432894522_ns4_test-17314328406863 table=ns4:test-17314328406863 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T17:34:54,943 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a2efbf6b09e33aa73d99cb054d7d7f3e}] 2024-11-12T17:34:54,944 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a2efbf6b09e33aa73d99cb054d7d7f3e 2024-11-12T17:34:55,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39493 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2024-11-12T17:34:55,096 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. 2024-11-12T17:34:55,096 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for a2efbf6b09e33aa73d99cb054d7d7f3e: 2024-11-12T17:34:55,096 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. for snapshot_1731432894522_ns4_test-17314328406863 completed. 2024-11-12T17:34:55,097 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(241): Storing 'ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e.' region-info for snapshot=snapshot_1731432894522_ns4_test-17314328406863 2024-11-12T17:34:55,097 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T17:34:55,097 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-12T17:34:55,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741921_1097 (size=52) 2024-11-12T17:34:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T17:34:55,452 INFO [master/1f32af3374c6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-12T17:34:55,452 INFO [master/1f32af3374c6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-12T17:34:55,503 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. 2024-11-12T17:34:55,503 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-12T17:34:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-12T17:34:55,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731432894522_ns4_test-17314328406863 on region a2efbf6b09e33aa73d99cb054d7d7f3e 2024-11-12T17:34:55,504 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a2efbf6b09e33aa73d99cb054d7d7f3e 2024-11-12T17:34:55,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-12T17:34:55,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; SnapshotRegionProcedure a2efbf6b09e33aa73d99cb054d7d7f3e in 562 msec 2024-11-12T17:34:55,506 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731432894522_ns4_test-17314328406863 table=ns4:test-17314328406863 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T17:34:55,507 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731432894522_ns4_test-17314328406863 table=ns4:test-17314328406863 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T17:34:55,508 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731432894522_ns4_test-17314328406863 table=ns4:test-17314328406863 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T17:34:55,508 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731432894522_ns4_test-17314328406863 2024-11-12T17:34:55,509 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_1731432894522_ns4_test-17314328406863 2024-11-12T17:34:55,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741922_1098 (size=443) 2024-11-12T17:34:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T17:34:55,919 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731432894522_ns4_test-17314328406863 table=ns4:test-17314328406863 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T17:34:55,925 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731432894522_ns4_test-17314328406863 table=ns4:test-17314328406863 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T17:34:55,925 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_1731432894522_ns4_test-17314328406863 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_1731432894522_ns4_test-17314328406863 2024-11-12T17:34:55,927 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731432894522_ns4_test-17314328406863 table=ns4:test-17314328406863 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T17:34:55,927 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731432894522_ns4_test-17314328406863 table=ns4:test-17314328406863 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-12T17:34:55,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731432894522_ns4_test-17314328406863 table=ns4:test-17314328406863 type=FLUSH ttl=0 } in 1.3960 sec 2024-11-12T17:34:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T17:34:56,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns4:test-17314328406863 completed 2024-11-12T17:34:56,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1731432896682_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } 2024-11-12T17:34:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731432896683 (current time:1731432896683). 2024-11-12T17:34:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T17:34:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731432896682_ns2_test-17314328406861 VERSION not specified, setting to 2 2024-11-12T17:34:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e59fc4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:56,685 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:56,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:56,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:56,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ea3462, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:56,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:56,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:56,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:56,686 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:56,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689751e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:56,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:56,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:56,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:56,691 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475. 2024-11-12T17:34:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T17:34:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:56,691 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:34:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T17:34:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731432896682_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } 2024-11-12T17:34:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731432896682_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-12T17:34:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T17:34:56,694 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731432896682_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T17:34:56,695 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731432896682_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T17:34:56,697 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731432896682_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T17:34:56,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741923_1099 (size=86) 2024-11-12T17:34:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T17:34:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T17:34:57,103 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731432896682_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T17:34:57,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6dd38527fc6c5b38bfed421a5b5be067}] 2024-11-12T17:34:57,104 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:57,221 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1a91481e8504b1563b1463c687c54efd, had cached 0 bytes from a total of 28925 2024-11-12T17:34:57,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39493 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2024-11-12T17:34:57,256 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:34:57,256 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 6dd38527fc6c5b38bfed421a5b5be067 1/1 column families, dataSize=170 B heapSize=816 B 2024-11-12T17:34:57,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/.tmp/f/1f3d76ee59374730a0a7334e164e7a05 is 38, key is row-t20/f:q1/1731432871014/Put/seqid=0 2024-11-12T17:34:57,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741924_1100 (size=5134) 2024-11-12T17:34:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T17:34:57,666 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/.tmp/f/1f3d76ee59374730a0a7334e164e7a05 2024-11-12T17:34:57,673 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/.tmp/f/1f3d76ee59374730a0a7334e164e7a05 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/1f3d76ee59374730a0a7334e164e7a05 2024-11-12T17:34:57,679 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/1f3d76ee59374730a0a7334e164e7a05, entries=5, sequenceid=111, filesize=5.0 K 2024-11-12T17:34:57,679 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for 6dd38527fc6c5b38bfed421a5b5be067 in 423ms, sequenceid=111, compaction requested=false 2024-11-12T17:34:57,680 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 6dd38527fc6c5b38bfed421a5b5be067: 2024-11-12T17:34:57,680 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. for snapshot_1731432896682_ns2_test-17314328406861 completed. 2024-11-12T17:34:57,680 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067.' region-info for snapshot=snapshot_1731432896682_ns2_test-17314328406861 2024-11-12T17:34:57,680 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T17:34:57,680 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/9f1293a0d2294237898b5e4d02393945, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/1f3d76ee59374730a0a7334e164e7a05] hfiles 2024-11-12T17:34:57,680 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/9f1293a0d2294237898b5e4d02393945 for snapshot=snapshot_1731432896682_ns2_test-17314328406861 2024-11-12T17:34:57,680 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/1f3d76ee59374730a0a7334e164e7a05 for snapshot=snapshot_1731432896682_ns2_test-17314328406861 2024-11-12T17:34:57,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741925_1101 (size=130) 2024-11-12T17:34:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T17:34:58,085 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:34:58,086 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-12T17:34:58,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-12T17:34:58,086 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731432896682_ns2_test-17314328406861 on region 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:58,087 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:34:58,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-12T17:34:58,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 6dd38527fc6c5b38bfed421a5b5be067 in 985 msec 2024-11-12T17:34:58,089 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731432896682_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T17:34:58,090 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731432896682_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T17:34:58,090 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731432896682_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T17:34:58,091 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731432896682_ns2_test-17314328406861 2024-11-12T17:34:58,091 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_1731432896682_ns2_test-17314328406861 2024-11-12T17:34:58,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741926_1102 (size=496) 2024-11-12T17:34:58,502 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731432896682_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T17:34:58,509 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731432896682_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T17:34:58,509 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_1731432896682_ns2_test-17314328406861 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_1731432896682_ns2_test-17314328406861 2024-11-12T17:34:58,511 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731432896682_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T17:34:58,511 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731432896682_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-12T17:34:58,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731432896682_ns2_test-17314328406861 table=ns2:test-17314328406861 type=FLUSH ttl=0 } in 1.8190 sec 2024-11-12T17:34:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T17:34:58,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17314328406861 completed 2024-11-12T17:34:58,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1731432898852_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } 2024-11-12T17:34:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731432898852 (current time:1731432898852). 2024-11-12T17:34:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T17:34:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731432898852_default_test-1731432840686 VERSION not specified, setting to 2 2024-11-12T17:34:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28af2eb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:34:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:34:58,854 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:34:58,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:34:58,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:34:58,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@578280ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:58,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:34:58,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:34:58,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:58,856 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:34:58,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29a7b220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:34:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:34:58,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:34:58,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:34:58,859 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:34:58,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475. 2024-11-12T17:34:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T17:34:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:34:58,860 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:34:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T17:34:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731432898852_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } 2024-11-12T17:34:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731432898852_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-12T17:34:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T17:34:58,863 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731432898852_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T17:34:58,864 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731432898852_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T17:34:58,866 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731432898852_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T17:34:58,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741927_1103 (size=84) 2024-11-12T17:34:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T17:34:59,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T17:34:59,272 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731432898852_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T17:34:59,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e}] 2024-11-12T17:34:59,273 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:59,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39493 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2024-11-12T17:34:59,425 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:59,426 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for f14197f5e90bb7bd826d4a3ae95e1d7e: 2024-11-12T17:34:59,426 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. for snapshot_1731432898852_default_test-1731432840686 completed. 2024-11-12T17:34:59,426 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(241): Storing 'test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e.' region-info for snapshot=snapshot_1731432898852_default_test-1731432840686 2024-11-12T17:34:59,426 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T17:34:59,426 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/13333c2c3c1a42179067d2c31f89d94b, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/fedf89305dee474fbf5f7539130b7d50] hfiles 2024-11-12T17:34:59,426 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/13333c2c3c1a42179067d2c31f89d94b for snapshot=snapshot_1731432898852_default_test-1731432840686 2024-11-12T17:34:59,426 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/fedf89305dee474fbf5f7539130b7d50 for snapshot=snapshot_1731432898852_default_test-1731432840686 2024-11-12T17:34:59,426 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-12T17:34:59,426 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/5c0434f3a70a41059b32fac676fc1ec2, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/751cbd875328413f90578c10809be672] hfiles 2024-11-12T17:34:59,426 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/5c0434f3a70a41059b32fac676fc1ec2 for snapshot=snapshot_1731432898852_default_test-1731432840686 2024-11-12T17:34:59,426 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/751cbd875328413f90578c10809be672 for snapshot=snapshot_1731432898852_default_test-1731432840686 2024-11-12T17:34:59,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741928_1104 (size=224) 2024-11-12T17:34:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T17:34:59,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:34:59,833 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-12T17:34:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-12T17:34:59,833 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731432898852_default_test-1731432840686 on region f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:59,834 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:59,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-12T17:34:59,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SnapshotRegionProcedure f14197f5e90bb7bd826d4a3ae95e1d7e in 562 msec 2024-11-12T17:34:59,836 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731432898852_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T17:34:59,837 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731432898852_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T17:34:59,838 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-12T17:34:59,838 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-12T17:34:59,838 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:59,839 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-12T17:34:59,839 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:59,839 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f2 2024-11-12T17:34:59,839 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:34:59,840 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/mobdir/data/default/test-1731432840686/af300c6268089b7a8d3f7acc33e67507/mob/d41d8cd98f00b204e9800998ecf8427e20241112171447fa3da54c0aa83bc727d43d6045_f14197f5e90bb7bd826d4a3ae95e1d7e] hfiles 2024-11-12T17:34:59,840 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/1): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/mobdir/data/default/test-1731432840686/af300c6268089b7a8d3f7acc33e67507/mob/d41d8cd98f00b204e9800998ecf8427e20241112171447fa3da54c0aa83bc727d43d6045_f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:34:59,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741929_1105 (size=168) 2024-11-12T17:35:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T17:35:00,248 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731432898852_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T17:35:00,248 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731432898852_default_test-1731432840686 2024-11-12T17:35:00,249 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_1731432898852_default_test-1731432840686 2024-11-12T17:35:00,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741930_1106 (size=1347) 2024-11-12T17:35:00,660 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731432898852_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T17:35:00,668 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731432898852_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T17:35:00,668 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_1731432898852_default_test-1731432840686 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_1731432898852_default_test-1731432840686 2024-11-12T17:35:00,669 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731432898852_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T17:35:00,669 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731432898852_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-12T17:35:00,671 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731432898852_default_test-1731432840686 table=test-1731432840686 type=FLUSH ttl=0 } in 1.8080 sec 2024-11-12T17:35:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T17:35:01,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1731432840686 completed 2024-11-12T17:35:01,011 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1731432885889 2024-11-12T17:35:01,012 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-12T17:35:01,012 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 5 snapshots to be copied. 2024-11-12T17:35:01,012 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731432885889_backup:system_bulk 2024-11-12T17:35:01,012 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731432890203_backup_system_bulk to hdfs://localhost:32879/backupUT/backup_1731432885889/backup/system_bulk/ 2024-11-12T17:35:01,012 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-12T17:35:01,044 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:32879, inputRoot=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 2024-11-12T17:35:01,044 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1613456041_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:32879/backupUT/backup_1731432885889/backup/system_bulk, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:32879/backupUT/backup_1731432885889/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1731432890203_backup_system_bulk 2024-11-12T17:35:01,047 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-12T17:35:01,051 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_1731432890203_backup_system_bulk to hdfs://localhost:32879/backupUT/backup_1731432885889/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1731432890203_backup_system_bulk 2024-11-12T17:35:01,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741931_1107 (size=76) 2024-11-12T17:35:01,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741932_1108 (size=702) 2024-11-12T17:35:01,463 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:01,463 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:01,464 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:02,450 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop-7374938959902426016.jar 2024-11-12T17:35:02,451 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:02,451 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:02,451 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:02,451 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:02,452 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:02,452 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:02,452 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:02,452 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:02,453 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:02,453 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T17:35:02,453 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T17:35:02,453 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T17:35:02,454 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T17:35:02,454 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T17:35:02,454 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T17:35:02,454 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T17:35:02,455 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T17:35:02,455 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T17:35:02,455 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T17:35:02,456 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T17:35:02,456 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:02,456 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:02,457 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:02,457 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:02,457 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:02,457 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:02,458 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:02,499 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T17:35:02,526 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731432890203_backup_system_bulk' hfile list 2024-11-12T17:35:02,616 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local400401801_0005_e300ab43-d4c9-4c57-a2a1-a3938b8f68c2/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-12T17:35:02,616 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local400401801_0005_e300ab43-d4c9-4c57-a2a1-a3938b8f68c2/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-12T17:35:03,656 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-12T17:35:03,657 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-12T17:35:03,662 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731432890203_backup_system_bulk 2024-11-12T17:35:03,662 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731432890203_backup_system_bulk finished. 2024-11-12T17:35:03,662 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731432885889_ns3:test-17314328406862 2024-11-12T17:35:03,662 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731432892361_ns3_test-17314328406862 to hdfs://localhost:32879/backupUT/backup_1731432885889/ns3/test-17314328406862/ 2024-11-12T17:35:03,662 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-12T17:35:03,692 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:32879, inputRoot=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 2024-11-12T17:35:03,692 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1613456041_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:32879/backupUT/backup_1731432885889/ns3/test-17314328406862, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:32879/backupUT/backup_1731432885889/ns3/test-17314328406862/.hbase-snapshot/.tmp/snapshot_1731432892361_ns3_test-17314328406862 2024-11-12T17:35:03,694 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-12T17:35:03,698 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_1731432892361_ns3_test-17314328406862 to hdfs://localhost:32879/backupUT/backup_1731432885889/ns3/test-17314328406862/.hbase-snapshot/.tmp/snapshot_1731432892361_ns3_test-17314328406862 2024-11-12T17:35:03,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741933_1109 (size=86) 2024-11-12T17:35:03,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741934_1110 (size=443) 2024-11-12T17:35:04,110 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:04,110 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:04,111 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:05,094 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop-13809523905717831233.jar 2024-11-12T17:35:05,095 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:05,095 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:05,095 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:05,096 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:05,096 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:05,096 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:05,096 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:05,097 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:05,097 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:05,097 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T17:35:05,097 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T17:35:05,098 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T17:35:05,098 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T17:35:05,098 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T17:35:05,098 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T17:35:05,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T17:35:05,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T17:35:05,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T17:35:05,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T17:35:05,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T17:35:05,100 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:05,100 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:05,100 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:05,101 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:05,101 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:05,101 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:05,101 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:05,142 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T17:35:05,169 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731432892361_ns3_test-17314328406862' hfile list 2024-11-12T17:35:05,261 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local1018088292_0006_aca75b05-f848-41c1-a9c9-c45d6078e76d/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-12T17:35:05,261 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local1018088292_0006_aca75b05-f848-41c1-a9c9-c45d6078e76d/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-12T17:35:06,301 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-12T17:35:06,302 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-12T17:35:06,307 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731432892361_ns3_test-17314328406862 2024-11-12T17:35:06,307 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731432892361_ns3_test-17314328406862 finished. 2024-11-12T17:35:06,307 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731432885889_ns4:test-17314328406863 2024-11-12T17:35:06,307 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731432894522_ns4_test-17314328406863 to hdfs://localhost:32879/backupUT/backup_1731432885889/ns4/test-17314328406863/ 2024-11-12T17:35:06,307 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-12T17:35:06,339 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:32879, inputRoot=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 2024-11-12T17:35:06,339 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1613456041_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:32879/backupUT/backup_1731432885889/ns4/test-17314328406863, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:32879/backupUT/backup_1731432885889/ns4/test-17314328406863/.hbase-snapshot/.tmp/snapshot_1731432894522_ns4_test-17314328406863 2024-11-12T17:35:06,341 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-12T17:35:06,345 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_1731432894522_ns4_test-17314328406863 to hdfs://localhost:32879/backupUT/backup_1731432885889/ns4/test-17314328406863/.hbase-snapshot/.tmp/snapshot_1731432894522_ns4_test-17314328406863 2024-11-12T17:35:06,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741935_1111 (size=86) 2024-11-12T17:35:06,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741936_1112 (size=443) 2024-11-12T17:35:06,759 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:06,759 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:06,760 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:07,753 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop-2711622202965923068.jar 2024-11-12T17:35:07,754 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:07,754 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:07,754 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:07,755 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:07,755 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:07,755 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:07,756 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:07,756 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:07,756 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:07,756 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T17:35:07,757 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T17:35:07,757 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T17:35:07,757 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T17:35:07,757 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T17:35:07,758 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T17:35:07,758 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T17:35:07,758 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T17:35:07,758 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T17:35:07,759 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T17:35:07,759 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T17:35:07,759 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:07,760 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:07,760 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:07,760 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:07,761 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:07,761 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:07,761 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:07,806 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T17:35:07,835 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731432894522_ns4_test-17314328406863' hfile list 2024-11-12T17:35:07,933 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local1876102760_0007_1d2ba427-40a2-4b77-aa00-d44552b0b479/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-12T17:35:07,933 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local1876102760_0007_1d2ba427-40a2-4b77-aa00-d44552b0b479/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-12T17:35:08,974 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-12T17:35:08,975 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-12T17:35:08,980 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731432894522_ns4_test-17314328406863 2024-11-12T17:35:08,980 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731432894522_ns4_test-17314328406863 finished. 2024-11-12T17:35:08,980 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731432885889_ns2:test-17314328406861 2024-11-12T17:35:08,980 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731432896682_ns2_test-17314328406861 to hdfs://localhost:32879/backupUT/backup_1731432885889/ns2/test-17314328406861/ 2024-11-12T17:35:08,980 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-12T17:35:09,009 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:32879, inputRoot=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 2024-11-12T17:35:09,009 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1613456041_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:32879/backupUT/backup_1731432885889/ns2/test-17314328406861, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:32879/backupUT/backup_1731432885889/ns2/test-17314328406861/.hbase-snapshot/.tmp/snapshot_1731432896682_ns2_test-17314328406861 2024-11-12T17:35:09,011 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-12T17:35:09,017 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_1731432896682_ns2_test-17314328406861 to hdfs://localhost:32879/backupUT/backup_1731432885889/ns2/test-17314328406861/.hbase-snapshot/.tmp/snapshot_1731432896682_ns2_test-17314328406861 2024-11-12T17:35:09,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741937_1113 (size=86) 2024-11-12T17:35:09,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741938_1114 (size=496) 2024-11-12T17:35:09,428 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:09,428 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:09,429 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:10,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop-12107389778133568017.jar 2024-11-12T17:35:10,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:10,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:10,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:10,425 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:10,425 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:10,425 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:10,425 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:10,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:10,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:10,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T17:35:10,427 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T17:35:10,427 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T17:35:10,427 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T17:35:10,428 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T17:35:10,428 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T17:35:10,428 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T17:35:10,429 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T17:35:10,429 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T17:35:10,429 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T17:35:10,429 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T17:35:10,430 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:10,430 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:10,430 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:10,431 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:10,431 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:10,431 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:10,431 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:10,474 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T17:35:10,503 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731432896682_ns2_test-17314328406861' hfile list 2024-11-12T17:35:10,505 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=13.2 K 2024-11-12T17:35:10,595 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local670470652_0008_e009a7b9-9e08-438f-821d-e50ebdf477be/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-12T17:35:10,595 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local670470652_0008_e009a7b9-9e08-438f-821d-e50ebdf477be/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-12T17:35:10,665 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-12T17:35:10,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741939_1115 (size=8413) 2024-11-12T17:35:11,109 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/9f1293a0d2294237898b5e4d02393945 output=hdfs://localhost:32879/backupUT/backup_1731432885889/ns2/test-17314328406861/archive/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/9f1293a0d2294237898b5e4d02393945 2024-11-12T17:35:11,110 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-12T17:35:11,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741940_1116 (size=5134) 2024-11-12T17:35:11,544 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/1f3d76ee59374730a0a7334e164e7a05 output=hdfs://localhost:32879/backupUT/backup_1731432885889/ns2/test-17314328406861/archive/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/f/1f3d76ee59374730a0a7334e164e7a05 2024-11-12T17:35:11,544 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5134 (5.0 K) time=0sec 0.011M/sec 2024-11-12T17:35:11,636 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-12T17:35:11,637 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-12T17:35:11,644 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731432896682_ns2_test-17314328406861 2024-11-12T17:35:11,645 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731432896682_ns2_test-17314328406861 finished. 2024-11-12T17:35:11,645 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731432885889_test-1731432840686 2024-11-12T17:35:11,645 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731432898852_default_test-1731432840686 to hdfs://localhost:32879/backupUT/backup_1731432885889/default/test-1731432840686/ 2024-11-12T17:35:11,645 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-12T17:35:11,679 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:32879, inputRoot=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6 2024-11-12T17:35:11,679 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1613456041_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:32879/backupUT/backup_1731432885889/default/test-1731432840686, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:32879/backupUT/backup_1731432885889/default/test-1731432840686/.hbase-snapshot/.tmp/snapshot_1731432898852_default_test-1731432840686 2024-11-12T17:35:11,682 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-12T17:35:11,688 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_1731432898852_default_test-1731432840686 to hdfs://localhost:32879/backupUT/backup_1731432885889/default/test-1731432840686/.hbase-snapshot/.tmp/snapshot_1731432898852_default_test-1731432840686 2024-11-12T17:35:11,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741942_1118 (size=84) 2024-11-12T17:35:11,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741941_1117 (size=1347) 2024-11-12T17:35:12,102 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:12,102 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:12,103 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:13,089 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop-5300608384480288441.jar 2024-11-12T17:35:13,090 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:13,090 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:13,090 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:13,091 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:13,091 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:13,091 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:13,091 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:13,092 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:13,092 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:13,092 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T17:35:13,092 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T17:35:13,092 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T17:35:13,093 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T17:35:13,093 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T17:35:13,093 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T17:35:13,093 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T17:35:13,094 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T17:35:13,094 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T17:35:13,094 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T17:35:13,094 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T17:35:13,095 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:13,095 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:13,095 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:13,095 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:13,095 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T17:35:13,096 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:13,096 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:13,136 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T17:35:13,162 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731432898852_default_test-1731432840686' hfile list 2024-11-12T17:35:13,164 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=43.0 K 2024-11-12T17:35:13,251 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local1473876353_0009_1c74d639-3bbd-42b4-84f9-aea19299affb/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-12T17:35:13,251 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/mapreduce.cluster.local.dir/job_local1473876353_0009_1c74d639-3bbd-42b4-84f9-aea19299affb/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-12T17:35:13,322 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-12T17:35:13,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741943_1119 (size=13760) 2024-11-12T17:35:13,763 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/751cbd875328413f90578c10809be672 output=hdfs://localhost:32879/backupUT/backup_1731432885889/default/test-1731432840686/archive/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/751cbd875328413f90578c10809be672 2024-11-12T17:35:13,763 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=13760 (13.4 K) time=0sec 0.030M/sec 2024-11-12T17:35:13,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741944_1120 (size=10796) 2024-11-12T17:35:14,196 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/fedf89305dee474fbf5f7539130b7d50 output=hdfs://localhost:32879/backupUT/backup_1731432885889/default/test-1731432840686/archive/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/fedf89305dee474fbf5f7539130b7d50 2024-11-12T17:35:14,196 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=10796 (10.5 K) time=0sec 0.024M/sec 2024-11-12T17:35:14,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741945_1121 (size=8413) 2024-11-12T17:35:14,637 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/13333c2c3c1a42179067d2c31f89d94b output=hdfs://localhost:32879/backupUT/backup_1731432885889/default/test-1731432840686/archive/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f/13333c2c3c1a42179067d2c31f89d94b 2024-11-12T17:35:14,637 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-12T17:35:14,662 WARN [LocalJobRunner Map Task Executor #0 {}] mapred.Task(640): Task status: "copied 5.5 K/5.5 K (100.0%) from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/mobdir/data/default/test-1731432840686/af300c6268089b7a8d3f7acc33e67507/mob/d41d8cd98f00b204e9800998ecf8427e20241112171447fa3da54c0aa83bc727d43d6045_f14197f5e90bb7bd826d4a3ae95e1d7e to hdfs://localhost:32879/backupUT/backup_1731432885889/default/test-1731432840686/archive/data/default/test-1731432840686/af300c6268089b7a8d3f7acc33e67507/mob/d41d8cd98f00b204e9800998ecf8427e20241112171447fa3da54c0aa83bc727d43d6045_f14197f5e90bb7bd826d4a3ae95e1d7e" truncated to max limit (512 characters) 2024-11-12T17:35:14,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741946_1122 (size=5591) 2024-11-12T17:35:15,071 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/mobdir/data/default/test-1731432840686/af300c6268089b7a8d3f7acc33e67507/mob/d41d8cd98f00b204e9800998ecf8427e20241112171447fa3da54c0aa83bc727d43d6045_f14197f5e90bb7bd826d4a3ae95e1d7e output=hdfs://localhost:32879/backupUT/backup_1731432885889/default/test-1731432840686/archive/data/default/test-1731432840686/af300c6268089b7a8d3f7acc33e67507/mob/d41d8cd98f00b204e9800998ecf8427e20241112171447fa3da54c0aa83bc727d43d6045_f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:35:15,071 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5591 (5.5 K) time=0sec 0.012M/sec 2024-11-12T17:35:15,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741947_1123 (size=5484) 2024-11-12T17:35:15,512 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/5c0434f3a70a41059b32fac676fc1ec2 output=hdfs://localhost:32879/backupUT/backup_1731432885889/default/test-1731432840686/archive/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/mob/5c0434f3a70a41059b32fac676fc1ec2 2024-11-12T17:35:15,512 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-11-12T17:35:16,294 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-12T17:35:16,295 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-12T17:35:16,303 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731432898852_default_test-1731432840686 2024-11-12T17:35:16,303 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731432898852_default_test-1731432840686 finished. 2024-11-12T17:35:16,303 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): backup:system_bulk 2024-11-12T17:35:16,303 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns3:test-17314328406862 2024-11-12T17:35:16,303 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns4:test-17314328406863 2024-11-12T17:35:16,303 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17314328406861 2024-11-12T17:35:16,303 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1731432840686 2024-11-12T17:35:16,313 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731432885889 2024-11-12T17:35:16,313 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-12T17:35:16,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741948_1124 (size=209) 2024-11-12T17:35:16,718 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:32879/backupUT/backup_1731432885889/.backup.manifest 2024-11-12T17:35:16,718 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1731432885889 finished: type=FULL,tablelist=backup:system_bulk;ns3:test-17314328406862;ns4:test-17314328406863;ns2:test-17314328406861;test-1731432840686,targetRootDir=hdfs://localhost:32879/backupUT,startts=1731432890082,completets=1731432916312,bytescopied=0 2024-11-12T17:35:16,718 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-12T17:35:16,718 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731432890203_backup_system_bulk 2024-11-12T17:35:16,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1731432890203_backup_system_bulk" type: DISABLED 2024-11-12T17:35:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731432890203_backup_system_bulk 2024-11-12T17:35:16,723 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731432890203_backup_system_bulk for backup backup_1731432885889 succeeded. 2024-11-12T17:35:16,723 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731432892361_ns3_test-17314328406862 2024-11-12T17:35:16,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1731432892361_ns3_test-17314328406862" type: DISABLED 2024-11-12T17:35:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731432892361_ns3_test-17314328406862 2024-11-12T17:35:16,726 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731432892361_ns3_test-17314328406862 for backup backup_1731432885889 succeeded. 2024-11-12T17:35:16,726 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731432894522_ns4_test-17314328406863 2024-11-12T17:35:16,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1731432894522_ns4_test-17314328406863" type: DISABLED 2024-11-12T17:35:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731432894522_ns4_test-17314328406863 2024-11-12T17:35:16,729 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731432894522_ns4_test-17314328406863 for backup backup_1731432885889 succeeded. 2024-11-12T17:35:16,729 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731432896682_ns2_test-17314328406861 2024-11-12T17:35:16,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1731432896682_ns2_test-17314328406861" type: DISABLED 2024-11-12T17:35:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731432896682_ns2_test-17314328406861 2024-11-12T17:35:16,732 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731432896682_ns2_test-17314328406861 for backup backup_1731432885889 succeeded. 2024-11-12T17:35:16,732 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731432898852_default_test-1731432840686 2024-11-12T17:35:16,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1731432898852_default_test-1731432840686" type: DISABLED 2024-11-12T17:35:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731432898852_default_test-1731432840686 2024-11-12T17:35:16,735 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731432898852_default_test-1731432840686 for backup backup_1731432885889 succeeded. 2024-11-12T17:35:16,735 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-12T17:35:16,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-12T17:35:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-12T17:35:16,741 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-12T17:35:16,743 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-12T17:35:16,748 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1731432885889 completed. 2024-11-12T17:35:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:35:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:35:19,071 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T17:35:21,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:35:21,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:35:21,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:35:21,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:35:21,811 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-12T17:35:21,815 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731432921772 2024-11-12T17:35:21,817 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1731432885889} 2024-11-12T17:35:21,817 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-12T17:35:21,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:35:21,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:35:21,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:35:21,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:35:21,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:35:21,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:35:21,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:35:21,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:35:21,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:35:21,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:35:21,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T17:35:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731432921904 (current time:1731432921904). 2024-11-12T17:35:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T17:35:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-12T17:35:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14fbc0c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:35:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:35:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:35:21,906 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:35:21,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:35:21,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:35:21,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d6224, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:35:21,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:35:21,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:35:21,908 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:21,908 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:35:21,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6181cbf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:35:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:35:21,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:35:21,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:35:21,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:35:21,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475. 2024-11-12T17:35:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T17:35:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:21,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:35:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T17:35:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T17:35:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-12T17:35:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T17:35:21,916 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T17:35:21,917 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T17:35:21,919 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T17:35:21,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741949_1125 (size=52) 2024-11-12T17:35:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T17:35:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T17:35:22,325 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T17:35:22,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd}] 2024-11-12T17:35:22,326 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:22,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39493 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2024-11-12T17:35:22,478 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:22,479 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 1a91481e8504b1563b1463c687c54efd 2/2 column families, dataSize=2.54 KB heapSize=4.22 KB 2024-11-12T17:35:22,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/meta/b4a99af12984400cacb88a3a8ee57c24 is 158, key is trslm:hdfs://localhost:32879/backupUT\x00ns2:test-17314328406861/meta:log-roll-map/1731432916308/Put/seqid=0 2024-11-12T17:35:22,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741950_1126 (size=6646) 2024-11-12T17:35:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T17:35:22,890 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/meta/b4a99af12984400cacb88a3a8ee57c24 2024-11-12T17:35:22,900 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/048ef9580eec4c94b550a5a72a8eba96 is 826, key is session:backup_1731432885889/session:context/1731432916742/Put/seqid=0 2024-11-12T17:35:22,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741951_1127 (size=5928) 2024-11-12T17:35:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T17:35:23,305 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/048ef9580eec4c94b550a5a72a8eba96 2024-11-12T17:35:23,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/meta/b4a99af12984400cacb88a3a8ee57c24 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/b4a99af12984400cacb88a3a8ee57c24 2024-11-12T17:35:23,317 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/b4a99af12984400cacb88a3a8ee57c24, entries=12, sequenceid=42, filesize=6.5 K 2024-11-12T17:35:23,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/048ef9580eec4c94b550a5a72a8eba96 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/048ef9580eec4c94b550a5a72a8eba96 2024-11-12T17:35:23,324 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/048ef9580eec4c94b550a5a72a8eba96, entries=2, sequenceid=42, filesize=5.8 K 2024-11-12T17:35:23,326 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~2.54 KB/2604, heapSize ~4.19 KB/4288, currentSize=0 B/0 for 1a91481e8504b1563b1463c687c54efd in 848ms, sequenceid=42, compaction requested=true 2024-11-12T17:35:23,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 1a91481e8504b1563b1463c687c54efd: 2024-11-12T17:35:23,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. for snapshot_backup_system completed. 2024-11-12T17:35:23,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.' region-info for snapshot=snapshot_backup_system 2024-11-12T17:35:23,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T17:35:23,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/dae3103a2869480c805494f0d642f915, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/6f005e3ac6844baa93a9096523aa1ee7, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/b4a99af12984400cacb88a3a8ee57c24] hfiles 2024-11-12T17:35:23,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/dae3103a2869480c805494f0d642f915 for snapshot=snapshot_backup_system 2024-11-12T17:35:23,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/6f005e3ac6844baa93a9096523aa1ee7 for snapshot=snapshot_backup_system 2024-11-12T17:35:23,327 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/b4a99af12984400cacb88a3a8ee57c24 for snapshot=snapshot_backup_system 2024-11-12T17:35:23,327 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cd67ce9189984d3e9f4c1dee67666da9, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/c625085d4bf443bcad8f22e2942797d3, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/048ef9580eec4c94b550a5a72a8eba96] hfiles 2024-11-12T17:35:23,327 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/4): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6 for snapshot=snapshot_backup_system 2024-11-12T17:35:23,327 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/4): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cd67ce9189984d3e9f4c1dee67666da9 for snapshot=snapshot_backup_system 2024-11-12T17:35:23,327 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/4): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/c625085d4bf443bcad8f22e2942797d3 for snapshot=snapshot_backup_system 2024-11-12T17:35:23,327 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (4/4): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/048ef9580eec4c94b550a5a72a8eba96 for snapshot=snapshot_backup_system 2024-11-12T17:35:23,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741952_1128 (size=330) 2024-11-12T17:35:23,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:23,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-12T17:35:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-12T17:35:23,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:23,734 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:23,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-12T17:35:23,737 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T17:35:23,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 1a91481e8504b1563b1463c687c54efd in 1.4100 sec 2024-11-12T17:35:23,738 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T17:35:23,738 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T17:35:23,738 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-12T17:35:23,739 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-12T17:35:23,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741953_1129 (size=967) 2024-11-12T17:35:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T17:35:24,152 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T17:35:24,160 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T17:35:24,161 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/snapshot_backup_system 2024-11-12T17:35:24,162 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T17:35:24,162 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-12T17:35:24,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2480 sec 2024-11-12T17:35:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T17:35:26,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-12T17:35:26,072 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1731432921772 started at 1731432926072. 2024-11-12T17:35:26,074 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1731432921772 has been started. 2024-11-12T17:35:26,079 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [backup:system_bulk, test-1731432840686, ns2:test-17314328406861, ns3:test-17314328406862, ns4:test-17314328406863] 2024-11-12T17:35:26,082 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1731432876689for backupID backup_1731432921772 2024-11-12T17:35:26,082 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-12T17:35:26,083 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-12T17:35:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-12T17:35:26,083 INFO [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-12T17:35:26,084 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-12T17:35:26,084 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-12T17:35:26,084 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-12T17:35:26,084 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T17:35:26,084 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T17:35:26,088 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T17:35:26,088 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T17:35:26,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T17:35:26,088 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,088 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-12T17:35:26,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-12T17:35:26,089 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T17:35:26,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-12T17:35:26,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T17:35:26,089 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-12T17:35:26,090 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-12T17:35:26,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-12T17:35:26,090 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-12T17:35:26,090 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-12T17:35:26,090 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-12T17:35:26,090 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-12T17:35:26,090 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '1f32af3374c6,39493,1731432832063' joining acquired barrier for procedure (rolllog) in zk 2024-11-12T17:35:26,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,092 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-12T17:35:26,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T17:35:26,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T17:35:26,092 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-12T17:35:26,092 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-12T17:35:26,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T17:35:26,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T17:35:26,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:35:26,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T17:35:26,094 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '1f32af3374c6,39493,1731432832063' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-12T17:35:26,094 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7253e3fa[Count = 0] remaining members to acquire global barrier 2024-11-12T17:35:26,094 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-12T17:35:26,094 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-12T17:35:26,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T17:35:26,095 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-12T17:35:26,095 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-12T17:35:26,096 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-12T17:35:26,096 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,096 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-12T17:35:26,096 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-12T17:35:26,096 DEBUG [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,096 INFO [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1731432890102 highest: 1731432890137 on 1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,096 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0:(num 1731432890102) roll requested 2024-11-12T17:35:26,110 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432926096, exclude list is [], retry=0 2024-11-12T17:35:26,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:35:26,114 INFO [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432890102 with entries=2, filesize=645 B; new WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432926096 2024-11-12T17:35:26,114 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:35:26,114 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432890102 is not closed yet, will try archiving it next time 2024-11-12T17:35:26,114 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 1f32af3374c6%2C39493%2C1731432832063.meta:.meta(num 1731432890121) roll requested 2024-11-12T17:35:26,114 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432856655 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432856655 2024-11-12T17:35:26,116 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432890102 not finished, retry = 0 2024-11-12T17:35:26,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741911_1087 (size=653) 2024-11-12T17:35:26,127 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432926114.meta, exclude list is [], retry=0 2024-11-12T17:35:26,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:35:26,131 INFO [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432890121.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432926114.meta 2024-11-12T17:35:26,131 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:35:26,131 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432890121.meta is not closed yet, will try archiving it next time 2024-11-12T17:35:26,131 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1:(num 1731432890137) roll requested 2024-11-12T17:35:26,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741912_1088 (size=93) 2024-11-12T17:35:26,132 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432890121.meta not finished, retry = 0 2024-11-12T17:35:26,144 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432926131, exclude list is [], retry=0 2024-11-12T17:35:26,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41947,DS-598c5b73-e17e-430f-b948-145aa5f7a106,DISK] 2024-11-12T17:35:26,148 INFO [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432890137 with entries=17, filesize=4.59 KB; new WAL /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432926131 2024-11-12T17:35:26,148 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41645:41645)] 2024-11-12T17:35:26,148 DEBUG [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432890137 is not closed yet, will try archiving it next time 2024-11-12T17:35:26,148 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432876689 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432876689 2024-11-12T17:35:26,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741913_1089 (size=4710) 2024-11-12T17:35:26,156 DEBUG [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-12T17:35:26,157 INFO [rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1731432926096 on 1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T17:35:26,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:35:26,172 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-12T17:35:26,173 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '1f32af3374c6,39493,1731432832063' in zk 2024-11-12T17:35:26,175 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-12T17:35:26,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,175 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T17:35:26,175 DEBUG [member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-12T17:35:26,175 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,175 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T17:35:26,175 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T17:35:26,176 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T17:35:26,176 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T17:35:26,176 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:35:26,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T17:35:26,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:35:26,178 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,179 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '1f32af3374c6,39493,1731432832063': 2024-11-12T17:35:26,179 INFO [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-12T17:35:26,179 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-12T17:35:26,179 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '1f32af3374c6,39493,1731432832063' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-12T17:35:26,179 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-12T17:35:26,179 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-12T17:35:26,179 INFO [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-12T17:35:26,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T17:35:26,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T17:35:26,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T17:35:26,181 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-12T17:35:26,181 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-12T17:35:26,181 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T17:35:26,181 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T17:35:26,181 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T17:35:26,181 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T17:35:26,182 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T17:35:26,182 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T17:35:26,182 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,182 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:35:26,182 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T17:35:26,182 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T17:35:26,182 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:35:26,183 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,183 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,183 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T17:35:26,184 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T17:35:26,184 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T17:35:26,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T17:35:26,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T17:35:26,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/1f32af3374c6,39493,1731432832063 2024-11-12T17:35:26,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T17:35:26,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T17:35:26,186 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T17:35:26,187 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T17:35:26,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T17:35:26,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T17:35:26,187 DEBUG [(1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T17:35:26,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-12T17:35:26,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-12T17:35:26,187 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T17:35:26,187 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T17:35:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-12T17:35:26,194 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {1f32af3374c6:39493=1731432876689} newestTimestamps: {1f32af3374c6:39493=1731432890137} 2024-11-12T17:35:26,195 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 1f32af3374c6%2C39493%2C1731432832063.meta.1731432835143.meta 2024-11-12T17:35:26,195 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 1f32af3374c6%2C39493%2C1731432832063.meta.1731432876668.meta 2024-11-12T17:35:26,195 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 1f32af3374c6%2C39493%2C1731432832063.meta.1731432890121.meta 2024-11-12T17:35:26,195 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 1f32af3374c6%2C39493%2C1731432832063.meta.1731432926114.meta 2024-11-12T17:35:26,195 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432890102 2024-11-12T17:35:26,195 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432926096 2024-11-12T17:35:26,195 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432890137 2024-11-12T17:35:26,196 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432926131 2024-11-12T17:35:26,196 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs/1f32af3374c6%2C39493%2C1731432832063.meta.1731432856684.meta 2024-11-12T17:35:26,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741957_1133 (size=640) 2024-11-12T17:35:26,216 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432890102 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432890102 2024-11-12T17:35:26,233 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432890121.meta to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs/1f32af3374c6%2C39493%2C1731432832063.meta.1731432890121.meta 2024-11-12T17:35:26,607 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:backup:system_bulk target: hdfs://localhost:32879/backupUT/backup_1731432921772/backup/system_bulk descriptor: 'backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:35:26,607 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-12T17:35:26,609 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table backup:system_bulk 2024-11-12T17:35:26,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741958_1134 (size=45) 2024-11-12T17:35:27,015 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table backup:system_bulk 2024-11-12T17:35:27,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741959_1135 (size=390) 2024-11-12T17:35:27,425 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns3:test-17314328406862 target: hdfs://localhost:32879/backupUT/backup_1731432921772/ns3/test-17314328406862 descriptor: 'ns3:test-17314328406862', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:35:27,426 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-12T17:35:27,428 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns3:test-17314328406862 2024-11-12T17:35:27,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741960_1136 (size=50) 2024-11-12T17:35:27,833 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns3:test-17314328406862 2024-11-12T17:35:27,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741961_1137 (size=390) 2024-11-12T17:35:28,244 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns4:test-17314328406863 target: hdfs://localhost:32879/backupUT/backup_1731432921772/ns4/test-17314328406863 descriptor: 'ns4:test-17314328406863', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:35:28,244 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-12T17:35:28,248 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns4:test-17314328406863 2024-11-12T17:35:28,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741962_1138 (size=50) 2024-11-12T17:35:28,653 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns4:test-17314328406863 2024-11-12T17:35:28,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741963_1139 (size=364) 2024-11-12T17:35:29,065 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17314328406861 target: hdfs://localhost:32879/backupUT/backup_1731432921772/ns2/test-17314328406861 descriptor: 'ns2:test-17314328406861', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:35:29,065 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-12T17:35:29,068 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17314328406861 2024-11-12T17:35:29,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741964_1140 (size=50) 2024-11-12T17:35:29,473 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17314328406861 2024-11-12T17:35:29,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741965_1141 (size=950) 2024-11-12T17:35:29,884 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1731432840686 target: hdfs://localhost:32879/backupUT/backup_1731432921772/default/test-1731432840686 descriptor: 'test-1731432840686', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T17:35:29,884 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-12T17:35:29,887 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1731432840686 2024-11-12T17:35:29,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741966_1142 (size=53) 2024-11-12T17:35:30,292 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1731432840686 2024-11-12T17:35:30,303 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:32879/backupUT/.tmp/backup_1731432921772 from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432890102;hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1.1731432890137 2024-11-12T17:35:30,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ed9dd99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:35:30,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:35:30,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:35:30,305 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:35:30,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:35:30,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:35:30,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d3476fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:35:30,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:35:30,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:35:30,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:30,307 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:35:30,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781e5301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:35:30,307 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T17:35:30,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 1f32af3374c6,39475,1731432830222 2024-11-12T17:35:30,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@68bd85a7 2024-11-12T17:35:30,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T17:35:30,309 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T17:35:30,312 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-12T17:35:30,312 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table backup:system_bulk 2024-11-12T17:35:30,313 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T17:35:30,313 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:35:30,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:35:30,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:35:30,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:35:30,316 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for backup:system_bulk: backup:system_bulk; 2024-11-12T17:35:30,316 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1731432840686 2024-11-12T17:35:30,316 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1731432840686,, stopping at row=test-1731432840686 ,, for max=2147483647 with caching=100 2024-11-12T17:35:30,318 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1731432840686: test-1731432840686; 2024-11-12T17:35:30,318 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17314328406861 2024-11-12T17:35:30,318 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17314328406861,, stopping at row=ns2:test-17314328406861 ,, for max=2147483647 with caching=100 2024-11-12T17:35:30,320 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17314328406861: ns2:test-17314328406861; 2024-11-12T17:35:30,320 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns3:test-17314328406862 2024-11-12T17:35:30,320 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns3:test-17314328406862,, stopping at row=ns3:test-17314328406862 ,, for max=2147483647 with caching=100 2024-11-12T17:35:30,321 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns3:test-17314328406862: ns3:test-17314328406862; 2024-11-12T17:35:30,321 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns4:test-17314328406863 2024-11-12T17:35:30,321 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns4:test-17314328406863,, stopping at row=ns4:test-17314328406863 ,, for max=2147483647 with caching=100 2024-11-12T17:35:30,323 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns4:test-17314328406863: ns4:test-17314328406863; 2024-11-12T17:35:30,323 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 5 reduce partitions to match current region count for all tables 2024-11-12T17:35:30,323 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/hbase-staging/partitions_242ed05e-0b92-4fe4-814b-8c9809aa317d 2024-11-12T17:35:30,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741967_1143 (size=324) 2024-11-12T17:35:30,730 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:30,730 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:30,730 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:30,734 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region f14197f5e90bb7bd826d4a3ae95e1d7e, had cached 0 bytes from a total of 38453 2024-11-12T17:35:31,720 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop-11551187371103554444.jar 2024-11-12T17:35:31,720 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:31,720 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:31,721 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:31,721 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:31,721 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:31,722 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:31,722 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:31,722 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:31,722 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:31,722 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T17:35:31,723 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T17:35:31,723 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T17:35:31,723 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T17:35:31,723 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T17:35:31,724 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T17:35:31,724 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T17:35:31,724 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T17:35:31,724 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T17:35:31,725 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T17:35:31,725 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T17:35:31,725 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:31,725 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:31,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:31,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:31,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:31,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-12T17:35:31,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T17:35:31,726 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: backup:system_bulk,default:test-1731432840686,ns2:test-17314328406861,ns3:test-17314328406862,ns4:test-17314328406863 2024-11-12T17:35:31,727 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T17:35:31,727 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T17:35:31,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:31,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:31,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:35:31,727 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T17:35:32,712 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop-9320112691081750877.jar 2024-11-12T17:35:32,724 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T17:35:32,751 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432890102 for WAL files 2024-11-12T17:35:32,752 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected exception in incremental-backup: incremental copy backup_1731432921772File hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432890102 does not exist. java.io.FileNotFoundException: File hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432890102 does not exist. at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1294) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1268) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1213) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1209) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listLocatedStatus(DistributedFileSystem.java:1227) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listLocatedStatus(FileSystem.java:2263) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getFiles(WALInputFormat.java:356) ~[hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:321) ~[hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:301) ~[hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:311) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:328) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:201) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1677) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1695) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:426) ~[hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T17:35:32,753 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1731432921772,startts=1731432926072,failedts=1731432932752,failedphase=PREPARE_INCREMENTAL,failedmessage=File hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432890102 does not exist. 2024-11-12T17:35:32,753 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-12T17:35:32,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable backup:system 2024-11-12T17:35:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=backup:system 2024-11-12T17:35:32,769 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432932769"}]},"ts":"1731432932769"} 2024-11-12T17:35:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T17:35:32,771 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLING in hbase:meta 2024-11-12T17:35:32,771 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set backup:system to state=DISABLING 2024-11-12T17:35:32,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=backup:system}] 2024-11-12T17:35:32,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=1a91481e8504b1563b1463c687c54efd, UNASSIGN}] 2024-11-12T17:35:32,778 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=1a91481e8504b1563b1463c687c54efd, UNASSIGN 2024-11-12T17:35:32,778 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=1a91481e8504b1563b1463c687c54efd, regionState=CLOSING, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:35:32,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=1a91481e8504b1563b1463c687c54efd, UNASSIGN because future has completed 2024-11-12T17:35:32,781 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T17:35:32,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a91481e8504b1563b1463c687c54efd, server=1f32af3374c6,39493,1731432832063}] 2024-11-12T17:35:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T17:35:32,933 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:32,933 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T17:35:32,933 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 1a91481e8504b1563b1463c687c54efd, disabling compactions & flushes 2024-11-12T17:35:32,933 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:32,933 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:32,933 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. after waiting 0 ms 2024-11-12T17:35:32,933 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:32,933 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 1a91481e8504b1563b1463c687c54efd 2/2 column families, dataSize=398 B heapSize=1.04 KB 2024-11-12T17:35:32,938 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/meta/117c7dff2d5c4fb0af4ec8b399ab02c3 is 103, key is rslogts:hdfs://localhost:32879/backupUT\x001f32af3374c6:39493/meta:rs-log-ts/1731432926171/Put/seqid=0 2024-11-12T17:35:32,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741968_1144 (size=5338) 2024-11-12T17:35:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T17:35:33,342 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=99 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/meta/117c7dff2d5c4fb0af4ec8b399ab02c3 2024-11-12T17:35:33,351 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/cee3b7a2a72b462698cf64b650196ee0 is 303, key is session:backup_1731432921772/session:context/1731432926073/Put/seqid=0 2024-11-12T17:35:33,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741969_1145 (size=5391) 2024-11-12T17:35:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T17:35:33,691 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 6dd38527fc6c5b38bfed421a5b5be067, had cached 0 bytes from a total of 13547 2024-11-12T17:35:33,756 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=299 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/cee3b7a2a72b462698cf64b650196ee0 2024-11-12T17:35:33,762 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/meta/117c7dff2d5c4fb0af4ec8b399ab02c3 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/117c7dff2d5c4fb0af4ec8b399ab02c3 2024-11-12T17:35:33,766 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/117c7dff2d5c4fb0af4ec8b399ab02c3, entries=1, sequenceid=47, filesize=5.2 K 2024-11-12T17:35:33,767 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/cee3b7a2a72b462698cf64b650196ee0 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cee3b7a2a72b462698cf64b650196ee0 2024-11-12T17:35:33,772 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cee3b7a2a72b462698cf64b650196ee0, entries=1, sequenceid=47, filesize=5.3 K 2024-11-12T17:35:33,773 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 1a91481e8504b1563b1463c687c54efd in 840ms, sequenceid=47, compaction requested=true 2024-11-12T17:35:33,777 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/recovered.edits/50.seqid, newMaxSeqId=50, maxSeqId=1 2024-11-12T17:35:33,777 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T17:35:33,778 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:33,778 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 1a91481e8504b1563b1463c687c54efd: Waiting for close lock at 1731432932933Running coprocessor pre-close hooks at 1731432932933Disabling compacts and flushes for region at 1731432932933Disabling writes for close at 1731432932933Obtaining lock to block concurrent updates at 1731432932933Preparing flush snapshotting stores in 1a91481e8504b1563b1463c687c54efd at 1731432932933Finished memstore snapshotting backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd., syncing WAL and waiting on mvcc, flushsize=dataSize=398, getHeapSize=1032, getOffHeapSize=0, getCellsCount=2 at 1731432932934 (+1 ms)Flushing stores of backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. at 1731432932934Flushing 1a91481e8504b1563b1463c687c54efd/meta: creating writer at 1731432932934Flushing 1a91481e8504b1563b1463c687c54efd/meta: appending metadata at 1731432932937 (+3 ms)Flushing 1a91481e8504b1563b1463c687c54efd/meta: closing flushed file at 1731432932937Flushing 1a91481e8504b1563b1463c687c54efd/session: creating writer at 1731432933348 (+411 ms)Flushing 1a91481e8504b1563b1463c687c54efd/session: appending metadata at 1731432933351 (+3 ms)Flushing 1a91481e8504b1563b1463c687c54efd/session: closing flushed file at 1731432933351Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c62c07d: reopening flushed file at 1731432933761 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d9f2273: reopening flushed file at 1731432933767 (+6 ms)Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 1a91481e8504b1563b1463c687c54efd in 840ms, sequenceid=47, compaction requested=true at 1731432933773 (+6 ms)Writing region close event to WAL at 1731432933774 (+1 ms)Running coprocessor post-close hooks at 1731432933777 (+3 ms)Closed at 1731432933778 (+1 ms) 2024-11-12T17:35:33,780 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:33,780 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=1a91481e8504b1563b1463c687c54efd, regionState=CLOSED 2024-11-12T17:35:33,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a91481e8504b1563b1463c687c54efd, server=1f32af3374c6,39493,1731432832063 because future has completed 2024-11-12T17:35:33,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-12T17:35:33,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 1a91481e8504b1563b1463c687c54efd, server=1f32af3374c6,39493,1731432832063 in 1.0020 sec 2024-11-12T17:35:33,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-12T17:35:33,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=1a91481e8504b1563b1463c687c54efd, UNASSIGN in 1.0080 sec 2024-11-12T17:35:33,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-12T17:35:33,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=backup:system in 1.0150 sec 2024-11-12T17:35:33,790 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432933789"}]},"ts":"1731432933789"} 2024-11-12T17:35:33,791 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLED in hbase:meta 2024-11-12T17:35:33,791 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set backup:system to state=DISABLED 2024-11-12T17:35:33,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=backup:system in 1.0290 sec 2024-11-12T17:35:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T17:35:33,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: backup:system completed 2024-11-12T17:35:33,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.RawAsyncHBaseAdmin(2177): Taking restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1731432933918 2024-11-12T17:35:33,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1731432933918 table=backup:system type=FLUSH ttl=0 } 2024-11-12T17:35:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731432933919 (current time:1731432933919). 2024-11-12T17:35:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T17:35:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot hbase-failsafe-snapshot_backup_system-1731432933918 VERSION not specified, setting to 2 2024-11-12T17:35:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6c7e15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:35:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ClusterIdFetcher(90): Going to request 1f32af3374c6,39475,-1 for getting cluster id 2024-11-12T17:35:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T17:35:33,920 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98d01804-71a2-4562-9aba-c6a02e64b7ec' 2024-11-12T17:35:33,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T17:35:33,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98d01804-71a2-4562-9aba-c6a02e64b7ec" 2024-11-12T17:35:33,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@698fdebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:35:33,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1f32af3374c6,39475,-1] 2024-11-12T17:35:33,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T17:35:33,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:33,922 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T17:35:33,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@791d79ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T17:35:33,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T17:35:33,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1f32af3374c6,39493,1731432832063, seqNum=-1] 2024-11-12T17:35:33,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T17:35:33,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T17:35:33,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475. 2024-11-12T17:35:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T17:35:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:33,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:35:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T17:35:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731432933918 table=backup:system type=FLUSH ttl=0 } 2024-11-12T17:35:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731432933918 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-11-12T17:35:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T17:35:33,928 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731432933918 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T17:35:33,929 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731432933918 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T17:35:33,931 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731432933918 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T17:35:33,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741970_1146 (size=81) 2024-11-12T17:35:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T17:35:34,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T17:35:34,337 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731432933918 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2024-11-12T17:35:34,338 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-12T17:35:34,338 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-12T17:35:34,338 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:35:34,340 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/117c7dff2d5c4fb0af4ec8b399ab02c3, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/6f005e3ac6844baa93a9096523aa1ee7, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/b4a99af12984400cacb88a3a8ee57c24, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/dae3103a2869480c805494f0d642f915] hfiles 2024-11-12T17:35:34,340 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/4): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/117c7dff2d5c4fb0af4ec8b399ab02c3 2024-11-12T17:35:34,340 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/4): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/6f005e3ac6844baa93a9096523aa1ee7 2024-11-12T17:35:34,341 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/4): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/b4a99af12984400cacb88a3a8ee57c24 2024-11-12T17:35:34,341 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/4): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/dae3103a2869480c805494f0d642f915 2024-11-12T17:35:34,341 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:35:34,343 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/048ef9580eec4c94b550a5a72a8eba96, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/c625085d4bf443bcad8f22e2942797d3, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cd67ce9189984d3e9f4c1dee67666da9, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cee3b7a2a72b462698cf64b650196ee0] hfiles 2024-11-12T17:35:34,343 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/5): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/048ef9580eec4c94b550a5a72a8eba96 2024-11-12T17:35:34,343 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/5): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6 2024-11-12T17:35:34,343 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/5): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/c625085d4bf443bcad8f22e2942797d3 2024-11-12T17:35:34,343 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/5): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cd67ce9189984d3e9f4c1dee67666da9 2024-11-12T17:35:34,343 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (5/5): hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cee3b7a2a72b462698cf64b650196ee0 2024-11-12T17:35:34,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741971_1147 (size=409) 2024-11-12T17:35:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T17:35:34,748 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 1a91481e8504b1563b1463c687c54efd, NAME => 'backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2024-11-12T17:35:34,749 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731432933918 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T17:35:34,750 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731432933918 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T17:35:34,750 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1731432933918 2024-11-12T17:35:34,751 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1731432933918 2024-11-12T17:35:34,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741972_1148 (size=1046) 2024-11-12T17:35:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T17:35:35,161 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731432933918 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T17:35:35,170 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731432933918 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T17:35:35,171 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1731432933918 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1731432933918 2024-11-12T17:35:35,172 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731432933918 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T17:35:35,172 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731432933918 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-11-12T17:35:35,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731432933918 table=backup:system type=FLUSH ttl=0 } in 1.2460 sec 2024-11-12T17:35:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T17:35:36,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-12T17:35:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1731432921904 type: FLUSH version: 2 ttl: 0 ) 2024-11-12T17:35:36,094 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(1055): Restore snapshot=snapshot_backup_system as table=backup:system 2024-11-12T17:35:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-12T17:35:36,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741973_1149 (size=635) 2024-11-12T17:35:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-12T17:35:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-12T17:35:36,506 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2024-11-12T17:35:36,506 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2024-11-12T17:35:36,508 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(410): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T17:35:36,512 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(183): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1731432921904 type: FLUSH version: 2 ttl: 0 2024-11-12T17:35:36,513 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(850): get table regions: hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system 2024-11-12T17:35:36,515 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(862): found 1 regions for table=backup:system 2024-11-12T17:35:36,516 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(229): region to restore: 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:36,519 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:35:36,523 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/117c7dff2d5c4fb0af4ec8b399ab02c3 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/archive/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/117c7dff2d5c4fb0af4ec8b399ab02c3 2024-11-12T17:35:36,523 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:35:36,527 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cee3b7a2a72b462698cf64b650196ee0 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/archive/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cee3b7a2a72b462698cf64b650196ee0 2024-11-12T17:35:36,527 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(272): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1731432921904 type: FLUSH version: 2 ttl: 0 2024-11-12T17:35:36,529 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731432936528"}]},"ts":"9223372036854775807"} 2024-11-12T17:35:36,532 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T17:35:36,532 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1a91481e8504b1563b1463c687c54efd, NAME => 'backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T17:35:36,533 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1731432936529"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731432936529"}]},"ts":"1731432936529"} 2024-11-12T17:35:36,534 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T17:35:36,534 INFO [PEWorker-1 {}] assignment.RegionStateStore(585): Overwritten 1 regions to Meta 2024-11-12T17:35:36,534 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(586): Overwritten regions: [{ENCODED => 1a91481e8504b1563b1463c687c54efd, NAME => 'backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T17:35:36,535 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(481): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2024-11-12T17:35:36,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1731432921904 type: FLUSH version: 2 ttl: 0 ) in 451 msec 2024-11-12T17:35:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-12T17:35:36,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin(2211): Deleting restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1731432933918 2024-11-12T17:35:36,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "hbase-failsafe-snapshot_backup_system-1731432933918" type: DISABLED 2024-11-12T17:35:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(381): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1731432933918 2024-11-12T17:35:36,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.HMaster$12(2787): Client=jenkins//172.17.0.2 enable backup:system 2024-11-12T17:35:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:ENABLE_TABLE_PREPARE, hasLock=false; EnableTableProcedure table=backup:system 2024-11-12T17:35:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T17:35:36,737 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2024-11-12T17:35:36,738 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432936738"}]},"ts":"1731432936738"} 2024-11-12T17:35:36,739 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-12T17:35:36,740 DEBUG [PEWorker-2 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2024-11-12T17:35:36,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=1a91481e8504b1563b1463c687c54efd, ASSIGN}] 2024-11-12T17:35:36,741 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=1a91481e8504b1563b1463c687c54efd, ASSIGN 2024-11-12T17:35:36,742 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=1a91481e8504b1563b1463c687c54efd, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2024-11-12T17:35:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T17:35:36,893 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=1a91481e8504b1563b1463c687c54efd, regionState=OPENING, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:35:36,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=1a91481e8504b1563b1463c687c54efd, ASSIGN because future has completed 2024-11-12T17:35:36,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a91481e8504b1563b1463c687c54efd, server=1f32af3374c6,39493,1731432832063}] 2024-11-12T17:35:37,050 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(132): Open backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:37,050 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a91481e8504b1563b1463c687c54efd, NAME => 'backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.', STARTKEY => '', ENDKEY => ''} 2024-11-12T17:35:37,051 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T17:35:37,051 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:37,051 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(898): Instantiated backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T17:35:37,051 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7794): checking encryption for 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:37,051 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7797): checking classloading for 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:37,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T17:35:37,052 INFO [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:37,053 INFO [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a91481e8504b1563b1463c687c54efd columnFamilyName meta 2024-11-12T17:35:37,053 DEBUG [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:35:37,060 DEBUG [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/6f005e3ac6844baa93a9096523aa1ee7 2024-11-12T17:35:37,063 DEBUG [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/b4a99af12984400cacb88a3a8ee57c24 2024-11-12T17:35:37,067 DEBUG [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/dae3103a2869480c805494f0d642f915 2024-11-12T17:35:37,067 INFO [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] regionserver.HStore(327): Store=1a91481e8504b1563b1463c687c54efd/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:35:37,067 INFO [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:37,068 INFO [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a91481e8504b1563b1463c687c54efd columnFamilyName session 2024-11-12T17:35:37,068 DEBUG [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T17:35:37,074 DEBUG [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/048ef9580eec4c94b550a5a72a8eba96 2024-11-12T17:35:37,078 DEBUG [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6 2024-11-12T17:35:37,081 DEBUG [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/c625085d4bf443bcad8f22e2942797d3 2024-11-12T17:35:37,085 DEBUG [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cd67ce9189984d3e9f4c1dee67666da9 2024-11-12T17:35:37,085 INFO [StoreOpener-1a91481e8504b1563b1463c687c54efd-1 {}] regionserver.HStore(327): Store=1a91481e8504b1563b1463c687c54efd/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T17:35:37,085 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1038): replaying wal for 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:37,086 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:37,087 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:37,087 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1048): stopping wal replay for 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:37,088 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1060): Cleaning up temporary data for 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:37,088 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-12T17:35:37,089 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1093): writing seq id for 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:37,090 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1114): Opened 1a91481e8504b1563b1463c687c54efd; next sequenceid=51; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9829037760, jitterRate=-0.08459952473640442}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-12T17:35:37,090 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:37,090 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1006): Region open journal for 1a91481e8504b1563b1463c687c54efd: Running coprocessor pre-open hook at 1731432937051Writing region info on filesystem at 1731432937051Initializing all the Stores at 1731432937052 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432937052Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731432937052Cleaning up temporary data from old regions at 1731432937088 (+36 ms)Running coprocessor post-open hooks at 1731432937090 (+2 ms)Region opened successfully at 1731432937090 2024-11-12T17:35:37,091 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd., pid=70, masterSystemTime=1731432937047 2024-11-12T17:35:37,093 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store 1a91481e8504b1563b1463c687c54efd:meta, priority=-2147483648, current under compaction store size is 1 2024-11-12T17:35:37,095 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T17:35:37,095 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store 1a91481e8504b1563b1463c687c54efd:session, priority=-2147483648, current under compaction store size is 2 2024-11-12T17:35:37,095 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T17:35:37,095 DEBUG [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T17:35:37,095 DEBUG [RS:0;1f32af3374c6:39493-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T17:35:37,097 DEBUG [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:37,097 INFO [RS_OPEN_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(153): Opened backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:37,097 DEBUG [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18305 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T17:35:37,097 DEBUG [RS:0;1f32af3374c6:39493-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 23194 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T17:35:37,098 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=1a91481e8504b1563b1463c687c54efd, regionState=OPEN, openSeqNum=51, regionLocation=1f32af3374c6,39493,1731432832063 2024-11-12T17:35:37,099 DEBUG [RS:0;1f32af3374c6:39493-longCompactions-0 {}] regionserver.HStore(1541): 1a91481e8504b1563b1463c687c54efd/session is initiating minor compaction (all files) 2024-11-12T17:35:37,099 DEBUG [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] regionserver.HStore(1541): 1a91481e8504b1563b1463c687c54efd/meta is initiating minor compaction (all files) 2024-11-12T17:35:37,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a91481e8504b1563b1463c687c54efd, server=1f32af3374c6,39493,1731432832063 because future has completed 2024-11-12T17:35:37,100 INFO [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a91481e8504b1563b1463c687c54efd/meta in backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:37,100 INFO [RS:0;1f32af3374c6:39493-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a91481e8504b1563b1463c687c54efd/session in backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:37,100 INFO [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/dae3103a2869480c805494f0d642f915, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/6f005e3ac6844baa93a9096523aa1ee7, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/b4a99af12984400cacb88a3a8ee57c24] into tmpdir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp, totalSize=17.9 K 2024-11-12T17:35:37,100 INFO [RS:0;1f32af3374c6:39493-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cd67ce9189984d3e9f4c1dee67666da9, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/c625085d4bf443bcad8f22e2942797d3, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/048ef9580eec4c94b550a5a72a8eba96] into tmpdir=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp, totalSize=22.7 K 2024-11-12T17:35:37,101 DEBUG [RS:0;1f32af3374c6:39493-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fc13f1a42ee47e4b896a898f018a2c6, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731432854413 2024-11-12T17:35:37,101 DEBUG [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] compactions.Compactor(225): Compacting dae3103a2869480c805494f0d642f915, keycount=6, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731432856764 2024-11-12T17:35:37,102 DEBUG [RS:0;1f32af3374c6:39493-longCompactions-0 {}] compactions.Compactor(225): Compacting cd67ce9189984d3e9f4c1dee67666da9, keycount=2, bloomtype=ROW, size=5.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731432870625 2024-11-12T17:35:37,102 DEBUG [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f005e3ac6844baa93a9096523aa1ee7, keycount=4, bloomtype=ROW, size=5.6 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731432876746 2024-11-12T17:35:37,102 DEBUG [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4a99af12984400cacb88a3a8ee57c24, keycount=12, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731432890178 2024-11-12T17:35:37,103 DEBUG [RS:0;1f32af3374c6:39493-longCompactions-0 {}] compactions.Compactor(225): Compacting c625085d4bf443bcad8f22e2942797d3, keycount=3, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731432885203 2024-11-12T17:35:37,103 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-11-12T17:35:37,103 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a91481e8504b1563b1463c687c54efd, server=1f32af3374c6,39493,1731432832063 in 206 msec 2024-11-12T17:35:37,103 DEBUG [RS:0;1f32af3374c6:39493-longCompactions-0 {}] compactions.Compactor(225): Compacting 048ef9580eec4c94b550a5a72a8eba96, keycount=2, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731432916742 2024-11-12T17:35:37,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-12T17:35:37,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=1a91481e8504b1563b1463c687c54efd, ASSIGN in 363 msec 2024-11-12T17:35:37,106 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731432937106"}]},"ts":"1731432937106"} 2024-11-12T17:35:37,107 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-12T17:35:37,107 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2024-11-12T17:35:37,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; EnableTableProcedure table=backup:system in 377 msec 2024-11-12T17:35:37,130 INFO [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a91481e8504b1563b1463c687c54efd#meta#compaction#16 average throughput is 1.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T17:35:37,130 DEBUG [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/meta/1cb7aec2a4604e55a99445194ff745cc is 158, key is trslm:hdfs://localhost:32879/backupUT\x00ns2:test-17314328406861/meta:log-roll-map/1731432916308/Put/seqid=0 2024-11-12T17:35:37,133 INFO [RS:0;1f32af3374c6:39493-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a91481e8504b1563b1463c687c54efd#session#compaction#17 average throughput is 2.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T17:35:37,134 DEBUG [RS:0;1f32af3374c6:39493-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/6d611bacd4954c9fb77cb1aca572e0fc is 1245, key is session:backup_1731432885797/session:context/1731432885885/Put/seqid=0 2024-11-12T17:35:37,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741974_1150 (size=6749) 2024-11-12T17:35:37,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741975_1151 (size=8073) 2024-11-12T17:35:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T17:35:37,362 DEBUG [Time-limited test {}] impl.BackupSystemTable(1544): Done restoring backup system table 2024-11-12T17:35:37,362 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: ENABLE, Table Name: backup:system completed 2024-11-12T17:35:37,362 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-12T17:35:37,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-12T17:35:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39475 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-12T17:35:37,370 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-12T17:35:37,370 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: PREPARE_INCREMENTAL 2024-11-12T17:35:37,371 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-12T17:35:37,376 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1731432921772 failed. 2024-11-12T17:35:37,376 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T17:35:37,376 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:103) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T17:35:37,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:37,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:37,377 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:35:37,396 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=627 (was 610) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/cluster_5efe2478-d756-86b3-13f2-642347e6bab0/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/meta/1cb7aec2a4604e55a99445194ff745cc java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2101155470-172.17.0.2-1731432826388:blk_1073741954_1130, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'JobTracker' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(1f32af3374c6,39493,1731432832063)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/cluster_5efe2478-d756-86b3-13f2-642347e6bab0/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-482410819_20 at /127.0.0.1:55594 [Receiving block BP-2101155470-172.17.0.2-1731432826388:blk_1073741955_1131] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-482410819_20 at /127.0.0.1:55580 [Receiving block BP-2101155470-172.17.0.2-1731432826388:blk_1073741954_1130] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/1f32af3374c6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1415601277_20 at /127.0.0.1:53332 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2101155470-172.17.0.2-1731432826388:blk_1073741956_1132, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-482410819_20 at /127.0.0.1:55604 [Receiving block BP-2101155470-172.17.0.2-1731432826388:blk_1073741956_1132] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: member: '1f32af3374c6,39493,1731432832063' subprocedure-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/6d611bacd4954c9fb77cb1aca572e0fc java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: RS:0;1f32af3374c6:39493-shortCompactions-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:1013) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:949) app//org.apache.hadoop.hdfs.DFSOutputStream.closeImpl(DFSOutputStream.java:913) app//org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) app//org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) app//org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.finishClose(HFileWriterImpl.java:859) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.close(HFileWriterImpl.java:680) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter$SingleStoreFileWriter.close(StoreFileWriter.java:787) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter.close(StoreFileWriter.java:294) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:70) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:38) app//org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:398) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:62) app//org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:122) app//org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1181) app//org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2424) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:667) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:716) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2101155470-172.17.0.2-1731432826388:blk_1073741955_1131, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: (1f32af3374c6,39475,1731432830222)-proc-coordinator-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=637 (was 633) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=81 (was 185), ProcessCount=13 (was 13), AvailableMemoryMB=8767 (was 9233) 2024-11-12T17:35:37,398 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=627 is superior to 500 2024-11-12T17:35:37,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T17:35:37,404 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T17:35:37,404 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T17:35:37,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:37,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:37,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:35:37,405 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T17:35:37,405 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=278140077, stopped=false 2024-11-12T17:35:37,405 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1f32af3374c6,39475,1731432830222 2024-11-12T17:35:37,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-12T17:35:37,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-12T17:35:37,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:35:37,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:35:37,407 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T17:35:37,408 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T17:35:37,408 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T17:35:37,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-12T17:35:37,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:37,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-12T17:35:37,409 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1f32af3374c6,39493,1731432832063' ***** 2024-11-12T17:35:37,409 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T17:35:37,409 INFO [RS:0;1f32af3374c6:39493 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-12T17:35:37,415 INFO [RS:0;1f32af3374c6:39493 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@6e12a44d{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-12T17:35:37,417 INFO [RS:0;1f32af3374c6:39493 {}] server.AbstractConnector(383): Stopped ServerConnector@5804e71e{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-12T17:35:37,417 INFO [RS:0;1f32af3374c6:39493 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T17:35:37,418 INFO [RS:0;1f32af3374c6:39493 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@23a380bc{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-12T17:35:37,418 INFO [RS:0;1f32af3374c6:39493 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@13c18cd6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,STOPPED} 2024-11-12T17:35:37,420 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T17:35:37,420 INFO [RS:0;1f32af3374c6:39493 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T17:35:37,420 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T17:35:37,420 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2024-11-12T17:35:37,420 INFO [RS:0;1f32af3374c6:39493 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T17:35:37,421 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(3091): Received CLOSE for 6dd38527fc6c5b38bfed421a5b5be067 2024-11-12T17:35:37,421 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(3091): Received CLOSE for a2efbf6b09e33aa73d99cb054d7d7f3e 2024-11-12T17:35:37,421 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(3091): Received CLOSE for 569aa6aeafd4b9d079658f569537f60c 2024-11-12T17:35:37,421 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(3091): Received CLOSE for 1a91481e8504b1563b1463c687c54efd 2024-11-12T17:35:37,421 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(3091): Received CLOSE for ad7fb5b314f3c7778103a7cc5221da50 2024-11-12T17:35:37,421 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(3091): Received CLOSE for f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:35:37,421 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(959): stopping server 1f32af3374c6,39493,1731432832063 2024-11-12T17:35:37,421 INFO [RS:0;1f32af3374c6:39493 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T17:35:37,421 INFO [RS:0;1f32af3374c6:39493 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1f32af3374c6:39493. 2024-11-12T17:35:37,421 DEBUG [RS:0;1f32af3374c6:39493 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T17:35:37,421 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6dd38527fc6c5b38bfed421a5b5be067, disabling compactions & flushes 2024-11-12T17:35:37,421 DEBUG [RS:0;1f32af3374c6:39493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:37,421 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:35:37,421 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:35:37,421 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. after waiting 0 ms 2024-11-12T17:35:37,421 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:35:37,421 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T17:35:37,426 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns2/test-17314328406861/6dd38527fc6c5b38bfed421a5b5be067/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2024-11-12T17:35:37,426 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T17:35:37,427 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:35:37,427 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6dd38527fc6c5b38bfed421a5b5be067: Waiting for close lock at 1731432937421Running coprocessor pre-close hooks at 1731432937421Disabling compacts and flushes for region at 1731432937421Disabling writes for close at 1731432937421Writing region close event to WAL at 1731432937422 (+1 ms)Running coprocessor post-close hooks at 1731432937426 (+4 ms)Closed at 1731432937426 2024-11-12T17:35:37,427 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067. 2024-11-12T17:35:37,427 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a2efbf6b09e33aa73d99cb054d7d7f3e, disabling compactions & flushes 2024-11-12T17:35:37,427 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. 2024-11-12T17:35:37,427 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. 2024-11-12T17:35:37,427 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. after waiting 0 ms 2024-11-12T17:35:37,427 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. 2024-11-12T17:35:37,430 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns4/test-17314328406863/a2efbf6b09e33aa73d99cb054d7d7f3e/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-12T17:35:37,431 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T17:35:37,431 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. 2024-11-12T17:35:37,431 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a2efbf6b09e33aa73d99cb054d7d7f3e: Waiting for close lock at 1731432937427Running coprocessor pre-close hooks at 1731432937427Disabling compacts and flushes for region at 1731432937427Disabling writes for close at 1731432937427Writing region close event to WAL at 1731432937427Running coprocessor post-close hooks at 1731432937431 (+4 ms)Closed at 1731432937431 2024-11-12T17:35:37,431 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17314328406863,,1731432847033.a2efbf6b09e33aa73d99cb054d7d7f3e. 2024-11-12T17:35:37,431 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 569aa6aeafd4b9d079658f569537f60c, disabling compactions & flushes 2024-11-12T17:35:37,431 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. 2024-11-12T17:35:37,431 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. 2024-11-12T17:35:37,431 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. after waiting 0 ms 2024-11-12T17:35:37,431 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. 2024-11-12T17:35:37,434 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system_bulk/569aa6aeafd4b9d079658f569537f60c/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-12T17:35:37,434 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T17:35:37,435 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. 2024-11-12T17:35:37,435 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 569aa6aeafd4b9d079658f569537f60c: Waiting for close lock at 1731432937431Running coprocessor pre-close hooks at 1731432937431Disabling compacts and flushes for region at 1731432937431Disabling writes for close at 1731432937431Writing region close event to WAL at 1731432937431Running coprocessor post-close hooks at 1731432937434 (+3 ms)Closed at 1731432937435 (+1 ms) 2024-11-12T17:35:37,435 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1731432853230.569aa6aeafd4b9d079658f569537f60c. 2024-11-12T17:35:37,435 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1a91481e8504b1563b1463c687c54efd, disabling compactions & flushes 2024-11-12T17:35:37,435 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:37,469 INFO [regionserver/1f32af3374c6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T17:35:37,499 INFO [regionserver/1f32af3374c6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T17:35:37,499 INFO [regionserver/1f32af3374c6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T17:35:37,541 DEBUG [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/meta/1cb7aec2a4604e55a99445194ff745cc as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/1cb7aec2a4604e55a99445194ff745cc 2024-11-12T17:35:37,544 DEBUG [RS:0;1f32af3374c6:39493-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/6d611bacd4954c9fb77cb1aca572e0fc as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/6d611bacd4954c9fb77cb1aca572e0fc 2024-11-12T17:35:37,553 DEBUG [RS:0;1f32af3374c6:39493-longCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-11-12T17:35:37,553 DEBUG [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-11-12T17:35:37,556 INFO [RS:0;1f32af3374c6:39493-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1a91481e8504b1563b1463c687c54efd/session of 1a91481e8504b1563b1463c687c54efd into 6d611bacd4954c9fb77cb1aca572e0fc(size=7.9 K), total size for store is 7.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T17:35:37,556 INFO [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a91481e8504b1563b1463c687c54efd/meta of 1a91481e8504b1563b1463c687c54efd into 1cb7aec2a4604e55a99445194ff745cc(size=6.6 K), total size for store is 6.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T17:35:37,556 DEBUG [RS:0;1f32af3374c6:39493-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a91481e8504b1563b1463c687c54efd: 2024-11-12T17:35:37,556 DEBUG [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a91481e8504b1563b1463c687c54efd: 2024-11-12T17:35:37,556 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:37,556 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:37,556 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. after waiting 0 ms 2024-11-12T17:35:37,556 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:37,556 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1a91481e8504b1563b1463c687c54efd 2/2 column families, dataSize=594 B heapSize=1.23 KB 2024-11-12T17:35:37,558 INFO [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd., storeName=1a91481e8504b1563b1463c687c54efd/meta, priority=13, startTime=1731432937093; duration=0sec 2024-11-12T17:35:37,558 INFO [RS:0;1f32af3374c6:39493-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd., storeName=1a91481e8504b1563b1463c687c54efd/session, priority=12, startTime=1731432937095; duration=0sec 2024-11-12T17:35:37,558 DEBUG [RS:0;1f32af3374c6:39493-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T17:35:37,558 DEBUG [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T17:35:37,558 DEBUG [RS:0;1f32af3374c6:39493-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a91481e8504b1563b1463c687c54efd:meta 2024-11-12T17:35:37,559 DEBUG [RS:0;1f32af3374c6:39493-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a91481e8504b1563b1463c687c54efd:session 2024-11-12T17:35:37,559 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T17:35:37,559 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T17:35:37,559 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T17:35:37,559 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-12T17:35:37,559 DEBUG [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T17:35:37,559 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 1a91481e8504b1563b1463c687c54efd=backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd., ad7fb5b314f3c7778103a7cc5221da50=ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50., f14197f5e90bb7bd826d4a3ae95e1d7e=test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e.} 2024-11-12T17:35:37,559 INFO [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T17:35:37,559 DEBUG [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T17:35:37,559 DEBUG [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T17:35:37,559 DEBUG [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T17:35:37,560 INFO [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.17 KB heapSize=23.40 KB 2024-11-12T17:35:37,560 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a91481e8504b1563b1463c687c54efd, ad7fb5b314f3c7778103a7cc5221da50, f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:35:37,561 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/3eb88a678188482f846437aa97606ff8 is 554, key is session:backup_1731432921772/session:context/1731432937371/Put/seqid=0 2024-11-12T17:35:37,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741976_1152 (size=5655) 2024-11-12T17:35:37,582 DEBUG [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/.tmp/info/58b9a0138d714c2d887a4bf24fbfdbea is 159, key is ns2:test-17314328406861,,1731432842526.6dd38527fc6c5b38bfed421a5b5be067./info:regioninfo/1731432843712/Put/seqid=0 2024-11-12T17:35:37,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741977_1153 (size=14126) 2024-11-12T17:35:37,760 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a91481e8504b1563b1463c687c54efd, ad7fb5b314f3c7778103a7cc5221da50, f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:35:37,960 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a91481e8504b1563b1463c687c54efd, ad7fb5b314f3c7778103a7cc5221da50, f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:35:37,970 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=594 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/3eb88a678188482f846437aa97606ff8 2024-11-12T17:35:37,976 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/.tmp/session/3eb88a678188482f846437aa97606ff8 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/3eb88a678188482f846437aa97606ff8 2024-11-12T17:35:37,981 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/3eb88a678188482f846437aa97606ff8, entries=2, sequenceid=57, filesize=5.5 K 2024-11-12T17:35:37,982 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for 1a91481e8504b1563b1463c687c54efd in 426ms, sequenceid=57, compaction requested=false 2024-11-12T17:35:37,983 DEBUG [StoreCloser-backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/dae3103a2869480c805494f0d642f915, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/6f005e3ac6844baa93a9096523aa1ee7, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/b4a99af12984400cacb88a3a8ee57c24] to archive 2024-11-12T17:35:37,983 DEBUG [StoreCloser-backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T17:35:37,987 DEBUG [StoreCloser-backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/dae3103a2869480c805494f0d642f915 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/archive/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/dae3103a2869480c805494f0d642f915 2024-11-12T17:35:37,987 INFO [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.16 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/.tmp/info/58b9a0138d714c2d887a4bf24fbfdbea 2024-11-12T17:35:37,988 DEBUG [StoreCloser-backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/6f005e3ac6844baa93a9096523aa1ee7 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/archive/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/6f005e3ac6844baa93a9096523aa1ee7 2024-11-12T17:35:37,989 DEBUG [StoreCloser-backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/b4a99af12984400cacb88a3a8ee57c24 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/archive/data/backup/system/1a91481e8504b1563b1463c687c54efd/meta/b4a99af12984400cacb88a3a8ee57c24 2024-11-12T17:35:37,994 DEBUG [StoreCloser-backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=1f32af3374c6:39475 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-12T17:35:37,995 WARN [StoreCloser-backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [dae3103a2869480c805494f0d642f915=5914, 6f005e3ac6844baa93a9096523aa1ee7=5745, b4a99af12984400cacb88a3a8ee57c24=6646] 2024-11-12T17:35:37,996 DEBUG [StoreCloser-backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cd67ce9189984d3e9f4c1dee67666da9, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/c625085d4bf443bcad8f22e2942797d3, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/048ef9580eec4c94b550a5a72a8eba96] to archive 2024-11-12T17:35:37,996 DEBUG [StoreCloser-backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T17:35:37,998 DEBUG [StoreCloser-backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/archive/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/0fc13f1a42ee47e4b896a898f018a2c6 2024-11-12T17:35:38,000 DEBUG [StoreCloser-backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cd67ce9189984d3e9f4c1dee67666da9 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/archive/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/cd67ce9189984d3e9f4c1dee67666da9 2024-11-12T17:35:38,001 DEBUG [StoreCloser-backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/c625085d4bf443bcad8f22e2942797d3 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/archive/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/c625085d4bf443bcad8f22e2942797d3 2024-11-12T17:35:38,002 DEBUG [StoreCloser-backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/048ef9580eec4c94b550a5a72a8eba96 to hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/archive/data/backup/system/1a91481e8504b1563b1463c687c54efd/session/048ef9580eec4c94b550a5a72a8eba96 2024-11-12T17:35:38,002 WARN [StoreCloser-backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [0fc13f1a42ee47e4b896a898f018a2c6=5049, cd67ce9189984d3e9f4c1dee67666da9=5535, c625085d4bf443bcad8f22e2942797d3=6682, 048ef9580eec4c94b550a5a72a8eba96=5928] 2024-11-12T17:35:38,005 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/backup/system/1a91481e8504b1563b1463c687c54efd/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=50 2024-11-12T17:35:38,006 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T17:35:38,006 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:38,006 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1a91481e8504b1563b1463c687c54efd: Waiting for close lock at 1731432937435Running coprocessor pre-close hooks at 1731432937435Disabling compacts and flushes for region at 1731432937435Disabling writes for close at 1731432937556 (+121 ms)Obtaining lock to block concurrent updates at 1731432937556Preparing flush snapshotting stores in 1a91481e8504b1563b1463c687c54efd at 1731432937556Finished memstore snapshotting backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd., syncing WAL and waiting on mvcc, flushsize=dataSize=594, getHeapSize=1224, getOffHeapSize=0, getCellsCount=2 at 1731432937556Flushing stores of backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. at 1731432937557 (+1 ms)Flushing 1a91481e8504b1563b1463c687c54efd/session: creating writer at 1731432937557Flushing 1a91481e8504b1563b1463c687c54efd/session: appending metadata at 1731432937561 (+4 ms)Flushing 1a91481e8504b1563b1463c687c54efd/session: closing flushed file at 1731432937561Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68fd462: reopening flushed file at 1731432937975 (+414 ms)Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for 1a91481e8504b1563b1463c687c54efd in 426ms, sequenceid=57, compaction requested=false at 1731432937982 (+7 ms)Writing region close event to WAL at 1731432938002 (+20 ms)Running coprocessor post-close hooks at 1731432938006 (+4 ms)Closed at 1731432938006 2024-11-12T17:35:38,006 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1731432851062.1a91481e8504b1563b1463c687c54efd. 2024-11-12T17:35:38,006 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ad7fb5b314f3c7778103a7cc5221da50, disabling compactions & flushes 2024-11-12T17:35:38,006 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. 2024-11-12T17:35:38,006 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. 2024-11-12T17:35:38,006 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. after waiting 0 ms 2024-11-12T17:35:38,006 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. 2024-11-12T17:35:38,009 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/ns3/test-17314328406862/ad7fb5b314f3c7778103a7cc5221da50/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-12T17:35:38,010 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T17:35:38,010 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. 2024-11-12T17:35:38,010 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ad7fb5b314f3c7778103a7cc5221da50: Waiting for close lock at 1731432938006Running coprocessor pre-close hooks at 1731432938006Disabling compacts and flushes for region at 1731432938006Disabling writes for close at 1731432938006Writing region close event to WAL at 1731432938006Running coprocessor post-close hooks at 1731432938010 (+4 ms)Closed at 1731432938010 2024-11-12T17:35:38,010 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17314328406862,,1731432844872.ad7fb5b314f3c7778103a7cc5221da50. 2024-11-12T17:35:38,010 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f14197f5e90bb7bd826d4a3ae95e1d7e, disabling compactions & flushes 2024-11-12T17:35:38,010 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:35:38,010 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:35:38,010 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. after waiting 0 ms 2024-11-12T17:35:38,010 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:35:38,010 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f14197f5e90bb7bd826d4a3ae95e1d7e 3/3 column families, dataSize=378 B heapSize=1.63 KB 2024-11-12T17:35:38,015 DEBUG [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/.tmp/ns/147795dbed5741fb9fe44bfe75a0a746 is 43, key is default/ns:d/1731432835308/Put/seqid=0 2024-11-12T17:35:38,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741978_1154 (size=5345) 2024-11-12T17:35:38,026 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/f2/96c38608d4eb43c3998da9c9ca7bed0d is 58, key is row-test-1731432840686-2-0/f2:q1/1731432916764/Put/seqid=0 2024-11-12T17:35:38,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741979_1155 (size=5464) 2024-11-12T17:35:38,160 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:35:38,163 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=1f32af3374c6:39475 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-12T17:35:38,361 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f14197f5e90bb7bd826d4a3ae95e1d7e 2024-11-12T17:35:38,420 INFO [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=235 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/.tmp/ns/147795dbed5741fb9fe44bfe75a0a746 2024-11-12T17:35:38,431 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=378 B at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/f2/96c38608d4eb43c3998da9c9ca7bed0d 2024-11-12T17:35:38,436 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/.tmp/f2/96c38608d4eb43c3998da9c9ca7bed0d as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f2/96c38608d4eb43c3998da9c9ca7bed0d 2024-11-12T17:35:38,441 DEBUG [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/.tmp/table/28d2898a47d44737aef6c7ff4b3b5fe0 is 59, key is ns2:test-17314328406861/table:state/1731432843724/Put/seqid=0 2024-11-12T17:35:38,441 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/f2/96c38608d4eb43c3998da9c9ca7bed0d, entries=7, sequenceid=348, filesize=5.3 K 2024-11-12T17:35:38,442 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for f14197f5e90bb7bd826d4a3ae95e1d7e in 432ms, sequenceid=348, compaction requested=false 2024-11-12T17:35:38,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741980_1156 (size=5866) 2024-11-12T17:35:38,447 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/default/test-1731432840686/f14197f5e90bb7bd826d4a3ae95e1d7e/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=337 2024-11-12T17:35:38,448 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T17:35:38,448 INFO [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:35:38,448 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f14197f5e90bb7bd826d4a3ae95e1d7e: Waiting for close lock at 1731432938010Running coprocessor pre-close hooks at 1731432938010Disabling compacts and flushes for region at 1731432938010Disabling writes for close at 1731432938010Obtaining lock to block concurrent updates at 1731432938010Preparing flush snapshotting stores in f14197f5e90bb7bd826d4a3ae95e1d7e at 1731432938010Finished memstore snapshotting test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e., syncing WAL and waiting on mvcc, flushsize=dataSize=378, getHeapSize=1616, getOffHeapSize=0, getCellsCount=7 at 1731432938010Flushing stores of test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. at 1731432938011 (+1 ms)Flushing f14197f5e90bb7bd826d4a3ae95e1d7e/f2: creating writer at 1731432938011Flushing f14197f5e90bb7bd826d4a3ae95e1d7e/f2: appending metadata at 1731432938026 (+15 ms)Flushing f14197f5e90bb7bd826d4a3ae95e1d7e/f2: closing flushed file at 1731432938026Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69f96872: reopening flushed file at 1731432938435 (+409 ms)Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for f14197f5e90bb7bd826d4a3ae95e1d7e in 432ms, sequenceid=348, compaction requested=false at 1731432938442 (+7 ms)Writing region close event to WAL at 1731432938444 (+2 ms)Running coprocessor post-close hooks at 1731432938448 (+4 ms)Closed at 1731432938448 2024-11-12T17:35:38,448 DEBUG [RS_CLOSE_REGION-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1731432840686,,1731432841189.f14197f5e90bb7bd826d4a3ae95e1d7e. 2024-11-12T17:35:38,561 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T17:35:38,561 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-12T17:35:38,561 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T17:35:38,761 DEBUG [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T17:35:38,847 INFO [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=800 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/.tmp/table/28d2898a47d44737aef6c7ff4b3b5fe0 2024-11-12T17:35:38,853 DEBUG [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/.tmp/info/58b9a0138d714c2d887a4bf24fbfdbea as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/info/58b9a0138d714c2d887a4bf24fbfdbea 2024-11-12T17:35:38,857 INFO [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/info/58b9a0138d714c2d887a4bf24fbfdbea, entries=69, sequenceid=59, filesize=13.8 K 2024-11-12T17:35:38,858 DEBUG [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/.tmp/ns/147795dbed5741fb9fe44bfe75a0a746 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/ns/147795dbed5741fb9fe44bfe75a0a746 2024-11-12T17:35:38,863 INFO [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/ns/147795dbed5741fb9fe44bfe75a0a746, entries=7, sequenceid=59, filesize=5.2 K 2024-11-12T17:35:38,864 DEBUG [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/.tmp/table/28d2898a47d44737aef6c7ff4b3b5fe0 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/table/28d2898a47d44737aef6c7ff4b3b5fe0 2024-11-12T17:35:38,869 INFO [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/table/28d2898a47d44737aef6c7ff4b3b5fe0, entries=13, sequenceid=59, filesize=5.7 K 2024-11-12T17:35:38,870 INFO [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1311ms, sequenceid=59, compaction requested=false 2024-11-12T17:35:38,874 DEBUG [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/data/hbase/meta/1588230740/recovered.edits/62.seqid, newMaxSeqId=62, maxSeqId=1 2024-11-12T17:35:38,875 DEBUG [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T17:35:38,875 DEBUG [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T17:35:38,875 INFO [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T17:35:38,875 DEBUG [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731432937559Running coprocessor pre-close hooks at 1731432937559Disabling compacts and flushes for region at 1731432937559Disabling writes for close at 1731432937559Obtaining lock to block concurrent updates at 1731432937560 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731432937560Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13486, getHeapSize=23896, getOffHeapSize=0, getCellsCount=125 at 1731432937560Flushing stores of hbase:meta,,1.1588230740 at 1731432937560Flushing 1588230740/info: creating writer at 1731432937560Flushing 1588230740/info: appending metadata at 1731432937582 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731432937582Flushing 1588230740/ns: creating writer at 1731432937993 (+411 ms)Flushing 1588230740/ns: appending metadata at 1731432938015 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1731432938015Flushing 1588230740/table: creating writer at 1731432938425 (+410 ms)Flushing 1588230740/table: appending metadata at 1731432938440 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731432938440Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54abb61a: reopening flushed file at 1731432938852 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fae93de: reopening flushed file at 1731432938857 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20310bd3: reopening flushed file at 1731432938863 (+6 ms)Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1311ms, sequenceid=59, compaction requested=false at 1731432938870 (+7 ms)Writing region close event to WAL at 1731432938871 (+1 ms)Running coprocessor post-close hooks at 1731432938875 (+4 ms)Closed at 1731432938875 2024-11-12T17:35:38,875 DEBUG [RS_CLOSE_META-regionserver/1f32af3374c6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T17:35:38,961 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(976): stopping server 1f32af3374c6,39493,1731432832063; all regions closed. 2024-11-12T17:35:38,965 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.meta.1731432926114.meta not finished, retry = 0 2024-11-12T17:35:38,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741955_1131 (size=3260) 2024-11-12T17:35:39,070 DEBUG [RS:0;1f32af3374c6:39493 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs 2024-11-12T17:35:39,070 INFO [RS:0;1f32af3374c6:39493 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1f32af3374c6%2C39493%2C1731432832063.meta:.meta(num 1731432926114) 2024-11-12T17:35:39,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741954_1130 (size=1037) 2024-11-12T17:35:39,072 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/WALs/1f32af3374c6,39493,1731432832063/1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0.1731432926096 not finished, retry = 0 2024-11-12T17:35:39,175 DEBUG [RS:0;1f32af3374c6:39493 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs 2024-11-12T17:35:39,175 INFO [RS:0;1f32af3374c6:39493 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-0:(num 1731432926096) 2024-11-12T17:35:39,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741956_1132 (size=5275) 2024-11-12T17:35:39,181 DEBUG [RS:0;1f32af3374c6:39493 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/oldWALs 2024-11-12T17:35:39,181 INFO [RS:0;1f32af3374c6:39493 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1f32af3374c6%2C39493%2C1731432832063.1f32af3374c6%2C39493%2C1731432832063.regiongroup-1:(num 1731432926131) 2024-11-12T17:35:39,181 DEBUG [RS:0;1f32af3374c6:39493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:39,181 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T17:35:39,181 INFO [RS:0;1f32af3374c6:39493 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T17:35:39,181 INFO [RS:0;1f32af3374c6:39493 {}] hbase.ChoreService(370): Chore service for: regionserver/1f32af3374c6:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-12T17:35:39,182 INFO [RS:0;1f32af3374c6:39493 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T17:35:39,182 INFO [regionserver/1f32af3374c6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T17:35:39,182 INFO [RS:0;1f32af3374c6:39493 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39493 2024-11-12T17:35:39,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/1f32af3374c6,39493,1731432832063 2024-11-12T17:35:39,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-12T17:35:39,186 INFO [RS:0;1f32af3374c6:39493 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T17:35:39,189 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1f32af3374c6,39493,1731432832063] 2024-11-12T17:35:39,190 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /1/draining/1f32af3374c6,39493,1731432832063 already deleted, retry=false 2024-11-12T17:35:39,191 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1f32af3374c6,39493,1731432832063 expired; onlineServers=0 2024-11-12T17:35:39,191 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1f32af3374c6,39475,1731432830222' ***** 2024-11-12T17:35:39,191 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T17:35:39,191 INFO [M:0;1f32af3374c6:39475 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-12T17:35:39,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2024-11-12T17:35:39,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T17:35:39,197 INFO [M:0;1f32af3374c6:39475 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@56e6bf8{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/master} 2024-11-12T17:35:39,197 INFO [M:0;1f32af3374c6:39475 {}] server.AbstractConnector(383): Stopped ServerConnector@69cb2ebd{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-12T17:35:39,197 INFO [M:0;1f32af3374c6:39475 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T17:35:39,197 INFO [M:0;1f32af3374c6:39475 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5b460d1c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-12T17:35:39,198 INFO [M:0;1f32af3374c6:39475 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@7ca98a11{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,STOPPED} 2024-11-12T17:35:39,198 INFO [M:0;1f32af3374c6:39475 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T17:35:39,198 INFO [M:0;1f32af3374c6:39475 {}] master.HMaster(1770): Stopping master jetty server 2024-11-12T17:35:39,199 INFO [M:0;1f32af3374c6:39475 {}] server.AbstractConnector(383): Stopped ServerConnector@4efbcd61{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-12T17:35:39,199 INFO [M:0;1f32af3374c6:39475 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T17:35:39,199 DEBUG [M:0;1f32af3374c6:39475 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T17:35:39,199 INFO [M:0;1f32af3374c6:39475 {}] master.BackupLogCleaner(194): Stopping BackupLogCleaner 2024-11-12T17:35:39,199 INFO [M:0;1f32af3374c6:39475 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;1f32af3374c6:39475. 2024-11-12T17:35:39,199 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T17:35:39,199 DEBUG [M:0;1f32af3374c6:39475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupHFileCleaner.stop(BackupHFileCleaner.java:156) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.cleanup(CleanerChore.java:444) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.cleanup(HFileCleaner.java:239) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:333) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:322) at org.apache.hadoop.hbase.master.HMaster.stopChores(HMaster.java:1923) at org.apache.hadoop.hbase.HBaseServerBase.stopChoreService(HBaseServerBase.java:439) at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1777) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T17:35:39,199 DEBUG [master/1f32af3374c6:0:becomeActiveMaster-HFileCleaner.large.0-1731432834330 {}] cleaner.HFileCleaner(306): Exit Thread[master/1f32af3374c6:0:becomeActiveMaster-HFileCleaner.large.0-1731432834330,5,FailOnTimeoutGroup] 2024-11-12T17:35:39,199 DEBUG [master/1f32af3374c6:0:becomeActiveMaster-HFileCleaner.small.0-1731432834331 {}] cleaner.HFileCleaner(306): Exit Thread[master/1f32af3374c6:0:becomeActiveMaster-HFileCleaner.small.0-1731432834331,5,FailOnTimeoutGroup] 2024-11-12T17:35:39,200 DEBUG [M:0;1f32af3374c6:39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:39,200 DEBUG [M:0;1f32af3374c6:39475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T17:35:39,200 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T17:35:39,200 DEBUG [M:0;1f32af3374c6:39475 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T17:35:39,200 INFO [M:0;1f32af3374c6:39475 {}] hbase.ChoreService(370): Chore service for: master/1f32af3374c6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T17:35:39,200 INFO [M:0;1f32af3374c6:39475 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T17:35:39,200 DEBUG [M:0;1f32af3374c6:39475 {}] master.HMaster(1795): Stopping service threads 2024-11-12T17:35:39,200 INFO [M:0;1f32af3374c6:39475 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T17:35:39,200 INFO [M:0;1f32af3374c6:39475 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T17:35:39,201 INFO [M:0;1f32af3374c6:39475 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T17:35:39,201 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T17:35:39,201 DEBUG [M:0;1f32af3374c6:39475 {}] zookeeper.ZKUtil(347): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2024-11-12T17:35:39,202 WARN [M:0;1f32af3374c6:39475 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T17:35:39,202 INFO [M:0;1f32af3374c6:39475 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/.lastflushedseqids 2024-11-12T17:35:39,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741981_1157 (size=495) 2024-11-12T17:35:39,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T17:35:39,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39493-0x1019d8bc6660001, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T17:35:39,290 INFO [RS:0;1f32af3374c6:39493 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T17:35:39,290 INFO [RS:0;1f32af3374c6:39493 {}] regionserver.HRegionServer(1031): Exiting; stopping=1f32af3374c6,39493,1731432832063; zookeeper connection closed. 2024-11-12T17:35:39,291 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@27ef07b1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@27ef07b1 2024-11-12T17:35:39,291 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-12T17:35:39,298 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-12T17:35:39,616 INFO [M:0;1f32af3374c6:39475 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T17:35:39,616 INFO [M:0;1f32af3374c6:39475 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T17:35:39,616 DEBUG [M:0;1f32af3374c6:39475 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T17:35:39,630 INFO [M:0;1f32af3374c6:39475 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T17:35:39,630 DEBUG [M:0;1f32af3374c6:39475 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T17:35:39,630 DEBUG [M:0;1f32af3374c6:39475 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T17:35:39,630 DEBUG [M:0;1f32af3374c6:39475 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T17:35:39,630 INFO [M:0;1f32af3374c6:39475 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=319.33 KB heapSize=375.52 KB 2024-11-12T17:35:39,645 DEBUG [M:0;1f32af3374c6:39475 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc7809741f2245ce9690ec749367d4a6 is 82, key is hbase:meta,,1/info:regioninfo/1731432835227/Put/seqid=0 2024-11-12T17:35:39,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741982_1158 (size=5672) 2024-11-12T17:35:40,050 INFO [M:0;1f32af3374c6:39475 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc7809741f2245ce9690ec749367d4a6 2024-11-12T17:35:40,072 DEBUG [M:0;1f32af3374c6:39475 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/159e4fe584f54cec94e2a537fdf77801 is 2184, key is \x00\x00\x00\x00\x00\x00\x00*/proc:d/1731432885794/Put/seqid=0 2024-11-12T17:35:40,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741983_1159 (size=26524) 2024-11-12T17:35:40,477 INFO [M:0;1f32af3374c6:39475 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=318.77 KB at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/159e4fe584f54cec94e2a537fdf77801 2024-11-12T17:35:40,482 INFO [M:0;1f32af3374c6:39475 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 159e4fe584f54cec94e2a537fdf77801 2024-11-12T17:35:40,497 DEBUG [M:0;1f32af3374c6:39475 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1cd544c3048d4e04884f6c70a16fbc06 is 69, key is 1f32af3374c6,39493,1731432832063/rs:state/1731432834387/Put/seqid=0 2024-11-12T17:35:40,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741984_1160 (size=5156) 2024-11-12T17:35:40,697 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T17:35:40,697 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17314328406863 2024-11-12T17:35:40,697 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-12T17:35:40,697 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1731432840686 2024-11-12T17:35:40,697 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T17:35:40,697 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-11-12T17:35:40,697 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17314328406862 2024-11-12T17:35:40,697 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17314328406861 2024-11-12T17:35:40,697 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T17:35:40,902 INFO [M:0;1f32af3374c6:39475 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1cd544c3048d4e04884f6c70a16fbc06 2024-11-12T17:35:40,908 DEBUG [M:0;1f32af3374c6:39475 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc7809741f2245ce9690ec749367d4a6 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cc7809741f2245ce9690ec749367d4a6 2024-11-12T17:35:40,912 INFO [M:0;1f32af3374c6:39475 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cc7809741f2245ce9690ec749367d4a6, entries=8, sequenceid=667, filesize=5.5 K 2024-11-12T17:35:40,913 DEBUG [M:0;1f32af3374c6:39475 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/159e4fe584f54cec94e2a537fdf77801 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/159e4fe584f54cec94e2a537fdf77801 2024-11-12T17:35:40,918 INFO [M:0;1f32af3374c6:39475 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 159e4fe584f54cec94e2a537fdf77801 2024-11-12T17:35:40,918 INFO [M:0;1f32af3374c6:39475 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/159e4fe584f54cec94e2a537fdf77801, entries=69, sequenceid=667, filesize=25.9 K 2024-11-12T17:35:40,919 DEBUG [M:0;1f32af3374c6:39475 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1cd544c3048d4e04884f6c70a16fbc06 as hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1cd544c3048d4e04884f6c70a16fbc06 2024-11-12T17:35:40,923 INFO [M:0;1f32af3374c6:39475 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32879/user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1cd544c3048d4e04884f6c70a16fbc06, entries=1, sequenceid=667, filesize=5.0 K 2024-11-12T17:35:40,924 INFO [M:0;1f32af3374c6:39475 {}] regionserver.HRegion(3140): Finished flush of dataSize ~319.33 KB/326989, heapSize ~375.23 KB/384232, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1294ms, sequenceid=667, compaction requested=false 2024-11-12T17:35:40,926 INFO [M:0;1f32af3374c6:39475 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T17:35:40,926 DEBUG [M:0;1f32af3374c6:39475 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731432939616Disabling compacts and flushes for region at 1731432939616Disabling writes for close at 1731432939630 (+14 ms)Obtaining lock to block concurrent updates at 1731432939630Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731432939630Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=326989, getHeapSize=384472, getOffHeapSize=0, getCellsCount=746 at 1731432939630Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731432939631 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731432939631Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731432939645 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731432939645Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731432940055 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731432940071 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731432940071Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731432940482 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731432940496 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731432940496Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d471fa0: reopening flushed file at 1731432940907 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a59e6a6: reopening flushed file at 1731432940912 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b4ebb1b: reopening flushed file at 1731432940918 (+6 ms)Finished flush of dataSize ~319.33 KB/326989, heapSize ~375.23 KB/384232, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1294ms, sequenceid=667, compaction requested=false at 1731432940924 (+6 ms)Writing region close event to WAL at 1731432940926 (+2 ms)Closed at 1731432940926 2024-11-12T17:35:40,928 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/95c1b5f0-3976-4165-2fde-fc633c7ce9f6/MasterData/WALs/1f32af3374c6,39475,1731432830222/1f32af3374c6%2C39475%2C1731432830222.1f32af3374c6%2C39475%2C1731432830222.regiongroup-0.1731432833488 not finished, retry = 0 2024-11-12T17:35:40,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41947 is added to blk_1073741830_1006 (size=371079) 2024-11-12T17:35:41,030 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T17:35:41,030 INFO [M:0;1f32af3374c6:39475 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T17:35:41,030 INFO [M:0;1f32af3374c6:39475 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2024-11-12T17:35:41,030 INFO [M:0;1f32af3374c6:39475 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39475 2024-11-12T17:35:41,031 INFO [M:0;1f32af3374c6:39475 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T17:35:41,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T17:35:41,133 INFO [M:0;1f32af3374c6:39475 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T17:35:41,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39475-0x1019d8bc6660000, quorum=127.0.0.1:61065, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T17:35:41,140 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1dce7e65{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T17:35:41,143 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2044050{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T17:35:41,143 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T17:35:41,143 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@19fb95db{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T17:35:41,143 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@9b2b78d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,STOPPED} 2024-11-12T17:35:41,145 WARN [BP-2101155470-172.17.0.2-1731432826388 heartbeating to localhost/127.0.0.1:32879 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T17:35:41,145 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T17:35:41,146 WARN [BP-2101155470-172.17.0.2-1731432826388 heartbeating to localhost/127.0.0.1:32879 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2101155470-172.17.0.2-1731432826388 (Datanode Uuid b7f84aa1-4646-4913-8f3f-fdf3fcbd413d) service to localhost/127.0.0.1:32879 2024-11-12T17:35:41,146 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T17:35:41,147 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/cluster_5efe2478-d756-86b3-13f2-642347e6bab0/data/data1/current/BP-2101155470-172.17.0.2-1731432826388 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T17:35:41,147 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/cluster_5efe2478-d756-86b3-13f2-642347e6bab0/data/data2/current/BP-2101155470-172.17.0.2-1731432826388 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T17:35:41,148 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T17:35:41,154 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6df1a11{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T17:35:41,155 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34c8bc74{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T17:35:41,155 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T17:35:41,155 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17280962{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T17:35:41,156 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71f46724{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,STOPPED} 2024-11-12T17:35:41,164 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T17:35:41,215 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-12T17:35:41,215 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2377): Stopping mini mapreduce cluster... 2024-11-12T17:35:41,221 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@29a5b7e8{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-12T17:35:41,221 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3b1e9d33{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T17:35:41,221 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T17:35:41,222 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7eed520b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-12T17:35:41,222 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2070a37b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,STOPPED} 2024-11-12T17:35:41,230 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@380ba3dd{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-12T17:35:41,231 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@279f726d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T17:35:41,231 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T17:35:41,231 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@444b3a56{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-12T17:35:41,231 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b88ca19{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,STOPPED} 2024-11-12T17:35:41,237 ERROR [Thread[Thread-270,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-12T17:35:41,237 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6428151{cluster,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-12T17:35:41,238 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@440bd171{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T17:35:41,238 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T17:35:41,238 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ccdffea{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-12T17:35:41,238 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@426811c2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,STOPPED} 2024-11-12T17:35:41,241 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2024-11-12T17:35:41,246 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(72): Returning, interrupted : java.lang.InterruptedException 2024-11-12T17:35:41,246 ERROR [ResourceManager Event Processor Monitor {}] resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessorMonitor(1193): Returning, interrupted : java.lang.InterruptedException: sleep interrupted 2024-11-12T17:35:41,248 ERROR [Thread[Thread-293,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-12T17:35:41,250 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@56fe9299{jobhistory,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-12T17:35:41,251 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1478f1aa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T17:35:41,251 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T17:35:41,251 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@338b53fb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-12T17:35:41,252 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a6206d8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/ec7a418c-8310-d01b-de9b-349e10dc469d/hadoop.log.dir/,STOPPED} 2024-11-12T17:35:41,253 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-12T17:35:41,253 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2380): Mini mapreduce cluster stopped