2024-12-05 15:47:11,272 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-12-05 15:47:11,287 main DEBUG Took 0.012857 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 15:47:11,288 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 15:47:11,288 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 15:47:11,289 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 15:47:11,290 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 15:47:11,305 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 15:47:11,321 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 15:47:11,323 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 15:47:11,323 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 15:47:11,324 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 15:47:11,324 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 15:47:11,325 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 15:47:11,326 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 15:47:11,327 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 15:47:11,327 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 15:47:11,328 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 15:47:11,329 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 15:47:11,329 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 15:47:11,330 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 15:47:11,330 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 15:47:11,331 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 15:47:11,331 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 15:47:11,332 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 15:47:11,332 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 15:47:11,332 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 15:47:11,333 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 15:47:11,333 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 15:47:11,334 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 15:47:11,334 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 15:47:11,335 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 15:47:11,335 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 15:47:11,336 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 15:47:11,338 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 15:47:11,339 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 15:47:11,341 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 15:47:11,342 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 15:47:11,344 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 15:47:11,345 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 15:47:11,355 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 15:47:11,359 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 15:47:11,360 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 15:47:11,361 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 15:47:11,362 main DEBUG createAppenders(={Console}) 2024-12-05 15:47:11,363 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 initialized 2024-12-05 15:47:11,363 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-12-05 15:47:11,363 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 OK. 2024-12-05 15:47:11,364 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 15:47:11,364 main DEBUG OutputStream closed 2024-12-05 15:47:11,365 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 15:47:11,365 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 15:47:11,365 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3c73951 OK 2024-12-05 15:47:11,449 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 15:47:11,451 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 15:47:11,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 15:47:11,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 15:47:11,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 15:47:11,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 15:47:11,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 15:47:11,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 15:47:11,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 15:47:11,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 15:47:11,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 15:47:11,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 15:47:11,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 15:47:11,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 15:47:11,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 15:47:11,458 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 15:47:11,458 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 15:47:11,459 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 15:47:11,461 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 15:47:11,462 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@61fe30) with optional ClassLoader: null 2024-12-05 15:47:11,462 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 15:47:11,463 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@61fe30] started OK. 2024-12-05T15:47:11,497 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackup timeout: 13 mins 2024-12-05 15:47:11,500 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 15:47:11,501 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T15:47:11,828 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986 2024-12-05T15:47:11,834 DEBUG [Time-limited test {}] impl.BackupManager(124): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-12-05T15:47:11,837 DEBUG [Time-limited test {}] impl.BackupManager(154): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:47:11,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T15:47:11,857 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/cluster_783e9d64-ce48-dc27-a242-83d4c51d286c, deleteOnExit=true 2024-12-05T15:47:11,857 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-05T15:47:11,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/test.cache.data in system properties and HBase conf 2024-12-05T15:47:11,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T15:47:11,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir in system properties and HBase conf 2024-12-05T15:47:11,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T15:47:11,862 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T15:47:11,862 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-05T15:47:11,977 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T15:47:12,104 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T15:47:12,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T15:47:12,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T15:47:12,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T15:47:12,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T15:47:12,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T15:47:12,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T15:47:12,113 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T15:47:12,114 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T15:47:12,114 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T15:47:12,115 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/nfs.dump.dir in system properties and HBase conf 2024-12-05T15:47:12,115 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/java.io.tmpdir in system properties and HBase conf 2024-12-05T15:47:12,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T15:47:12,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T15:47:12,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T15:47:13,127 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-05T15:47:13,216 INFO [Time-limited test {}] log.Log(170): Logging initialized @2705ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-05T15:47:13,304 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T15:47:13,373 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T15:47:13,402 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T15:47:13,403 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T15:47:13,405 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T15:47:13,426 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T15:47:13,430 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@205baae5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,AVAILABLE} 2024-12-05T15:47:13,432 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@325f8c88{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T15:47:13,625 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1d57f2ce{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/java.io.tmpdir/jetty-localhost-43893-hadoop-hdfs-3_4_1-tests_jar-_-any-13672916871317548028/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T15:47:13,655 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@382e7786{HTTP/1.1, (http/1.1)}{localhost:43893} 2024-12-05T15:47:13,655 INFO [Time-limited test {}] server.Server(415): Started @3145ms 2024-12-05T15:47:14,180 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T15:47:14,186 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T15:47:14,188 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T15:47:14,188 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T15:47:14,188 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T15:47:14,189 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@347c3eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,AVAILABLE} 2024-12-05T15:47:14,190 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@308dcebb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T15:47:14,282 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d3c17d4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/java.io.tmpdir/jetty-localhost-35617-hadoop-hdfs-3_4_1-tests_jar-_-any-16435363119587974007/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T15:47:14,283 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@575d15e0{HTTP/1.1, (http/1.1)}{localhost:35617} 2024-12-05T15:47:14,283 INFO [Time-limited test {}] server.Server(415): Started @3773ms 2024-12-05T15:47:14,341 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T15:47:15,197 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/cluster_783e9d64-ce48-dc27-a242-83d4c51d286c/data/data2/current/BP-671471983-172.17.0.3-1733413632680/current, will proceed with Du for space computation calculation, 2024-12-05T15:47:15,197 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/cluster_783e9d64-ce48-dc27-a242-83d4c51d286c/data/data1/current/BP-671471983-172.17.0.3-1733413632680/current, will proceed with Du for space computation calculation, 2024-12-05T15:47:15,222 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T15:47:15,269 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfc9901dfc5317a61 with lease ID 0x8b76234d3585583d: Processing first storage report for DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6 from datanode DatanodeRegistration(127.0.0.1:43303, datanodeUuid=8025ece3-eea0-4ad5-b9b3-0c5f4a722f8c, infoPort=34553, infoSecurePort=0, ipcPort=39423, storageInfo=lv=-57;cid=testClusterID;nsid=335635753;c=1733413632680) 2024-12-05T15:47:15,271 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfc9901dfc5317a61 with lease ID 0x8b76234d3585583d: from storage DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6 node DatanodeRegistration(127.0.0.1:43303, datanodeUuid=8025ece3-eea0-4ad5-b9b3-0c5f4a722f8c, infoPort=34553, infoSecurePort=0, ipcPort=39423, storageInfo=lv=-57;cid=testClusterID;nsid=335635753;c=1733413632680), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-05T15:47:15,271 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986 2024-12-05T15:47:15,271 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfc9901dfc5317a61 with lease ID 0x8b76234d3585583d: Processing first storage report for DS-6bc69fd9-c955-4286-b3f2-2841442ec3db from datanode DatanodeRegistration(127.0.0.1:43303, datanodeUuid=8025ece3-eea0-4ad5-b9b3-0c5f4a722f8c, infoPort=34553, infoSecurePort=0, ipcPort=39423, storageInfo=lv=-57;cid=testClusterID;nsid=335635753;c=1733413632680) 2024-12-05T15:47:15,272 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfc9901dfc5317a61 with lease ID 0x8b76234d3585583d: from storage DS-6bc69fd9-c955-4286-b3f2-2841442ec3db node DatanodeRegistration(127.0.0.1:43303, datanodeUuid=8025ece3-eea0-4ad5-b9b3-0c5f4a722f8c, infoPort=34553, infoSecurePort=0, ipcPort=39423, storageInfo=lv=-57;cid=testClusterID;nsid=335635753;c=1733413632680), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T15:47:15,360 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/cluster_783e9d64-ce48-dc27-a242-83d4c51d286c/zookeeper_0, clientPort=50094, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/cluster_783e9d64-ce48-dc27-a242-83d4c51d286c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/cluster_783e9d64-ce48-dc27-a242-83d4c51d286c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T15:47:15,371 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50094 2024-12-05T15:47:15,380 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T15:47:15,383 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T15:47:15,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741825_1001 (size=7) 2024-12-05T15:47:16,019 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 with version=8 2024-12-05T15:47:16,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging 2024-12-05T15:47:16,031 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(136): Setting Master Port to random. 2024-12-05T15:47:16,031 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(143): Setting RegionServer Port to random. 2024-12-05T15:47:16,031 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(153): Setting RS InfoServer Port to random. 2024-12-05T15:47:16,031 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(161): Setting Master InfoServer Port to random. 2024-12-05T15:47:16,110 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T15:47:16,354 INFO [Time-limited test {}] client.ConnectionUtils(128): master/5a29627a94df:0 server-side Connection retries=45 2024-12-05T15:47:16,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T15:47:16,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T15:47:16,370 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T15:47:16,370 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T15:47:16,370 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T15:47:16,519 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T15:47:16,575 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T15:47:16,584 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T15:47:16,587 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T15:47:16,611 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 148773 (auto-detected) 2024-12-05T15:47:16,612 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-05T15:47:16,630 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33117 2024-12-05T15:47:16,653 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33117 connecting to ZooKeeper ensemble=127.0.0.1:50094 2024-12-05T15:47:16,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:331170x0, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T15:47:16,747 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33117-0x101a75acab50000 connected 2024-12-05T15:47:16,822 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T15:47:16,824 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T15:47:16,833 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-05T15:47:16,855 INFO [Time-limited test {}] log.Log(170): Logging initialized @6345ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2024-12-05T15:47:16,893 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-12-05T15:47:16,893 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-12-05T15:47:16,894 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-12-05T15:47:16,894 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-12-05T15:47:16,895 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-12-05T15:47:16,896 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2024-12-05T15:47:16,896 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-12-05T15:47:16,897 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-12-05T15:47:16,898 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-12-05T15:47:16,951 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 39857 2024-12-05T15:47:16,952 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-12-05T15:47:16,964 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-05T15:47:16,966 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@4acbd630{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,AVAILABLE} 2024-12-05T15:47:16,966 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-05T15:47:16,967 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@ebfbfd2{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-12-05T15:47:16,999 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-12-05T15:47:17,006 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T15:47:17,006 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T15:47:17,007 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T15:47:17,011 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-05T15:47:17,530 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@77e4d37e{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-12-05T15:47:17,535 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1ab434da{HTTP/1.1, (http/1.1)}{0.0.0.0:39857} 2024-12-05T15:47:17,535 INFO [Time-limited test {}] server.Server(415): Started @7025ms 2024-12-05T15:47:17,538 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089, hbase.cluster.distributed=false 2024-12-05T15:47:17,560 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-12-05T15:47:17,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33117 2024-12-05T15:47:17,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33117 2024-12-05T15:47:17,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33117 2024-12-05T15:47:17,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33117 2024-12-05T15:47:17,567 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33117 2024-12-05T15:47:17,674 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5a29627a94df:0 server-side Connection retries=45 2024-12-05T15:47:17,676 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T15:47:17,676 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T15:47:17,676 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T15:47:17,677 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T15:47:17,677 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T15:47:17,680 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T15:47:17,682 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T15:47:17,683 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34815 2024-12-05T15:47:17,684 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34815 connecting to ZooKeeper ensemble=127.0.0.1:50094 2024-12-05T15:47:17,686 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T15:47:17,689 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T15:47:17,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:348150x0, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T15:47:17,763 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:348150x0, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-05T15:47:17,763 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34815-0x101a75acab50001 connected 2024-12-05T15:47:17,765 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-12-05T15:47:17,765 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-12-05T15:47:17,765 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-12-05T15:47:17,766 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-12-05T15:47:17,766 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-12-05T15:47:17,767 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2024-12-05T15:47:17,767 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-12-05T15:47:17,767 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-12-05T15:47:17,767 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-12-05T15:47:17,769 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 43439 2024-12-05T15:47:17,769 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-12-05T15:47:17,771 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-05T15:47:17,771 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@6e82f00d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,AVAILABLE} 2024-12-05T15:47:17,771 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-05T15:47:17,771 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5af9e42e{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-12-05T15:47:17,783 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-12-05T15:47:17,784 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T15:47:17,785 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T15:47:17,785 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T15:47:17,787 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-05T15:47:17,790 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@6edf2259{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-12-05T15:47:17,790 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1aa5419c{HTTP/1.1, (http/1.1)}{0.0.0.0:43439} 2024-12-05T15:47:17,790 INFO [Time-limited test {}] server.Server(415): Started @7280ms 2024-12-05T15:47:17,794 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T15:47:17,802 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T15:47:17,804 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2024-12-05T15:47:17,810 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-12-05T15:47:17,811 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34815 2024-12-05T15:47:17,811 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34815 2024-12-05T15:47:17,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34815 2024-12-05T15:47:17,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34815 2024-12-05T15:47:17,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34815 2024-12-05T15:47:17,827 DEBUG [M:0;5a29627a94df:33117 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;5a29627a94df:33117 2024-12-05T15:47:17,830 INFO [master/5a29627a94df:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-12-05T15:47:17,832 INFO [master/5a29627a94df:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@5207b9f1{HTTP/1.1, (http/1.1)}{0.0.0.0:32821} 2024-12-05T15:47:17,832 INFO [master/5a29627a94df:0:becomeActiveMaster {}] server.Server(415): Started @7322ms 2024-12-05T15:47:17,832 INFO [master/5a29627a94df:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /1/backup-masters/5a29627a94df,33117,1733413636177 2024-12-05T15:47:17,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-05T15:47:17,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-05T15:47:17,848 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/5a29627a94df,33117,1733413636177 2024-12-05T15:47:17,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2024-12-05T15:47:17,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:47:17,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:47:17,871 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on existing znode=/1/master 2024-12-05T15:47:17,872 INFO [master/5a29627a94df:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/5a29627a94df,33117,1733413636177 from backup master directory 2024-12-05T15:47:17,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-05T15:47:17,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/5a29627a94df,33117,1733413636177 2024-12-05T15:47:17,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-05T15:47:17,879 WARN [master/5a29627a94df:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T15:47:17,879 INFO [master/5a29627a94df:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=5a29627a94df,33117,1733413636177 2024-12-05T15:47:17,881 INFO [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T15:47:17,882 INFO [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T15:47:17,932 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase.id] with ID: 6ce2a395-fcd1-42e7-9dc1-a423aee17fc6 2024-12-05T15:47:17,932 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.tmp/hbase.id 2024-12-05T15:47:17,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741826_1002 (size=42) 2024-12-05T15:47:18,343 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.tmp/hbase.id]:[hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase.id] 2024-12-05T15:47:18,409 INFO [master/5a29627a94df:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T15:47:18,414 INFO [master/5a29627a94df:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T15:47:18,435 INFO [master/5a29627a94df:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-12-05T15:47:18,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:47:18,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:47:18,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741827_1003 (size=196) 2024-12-05T15:47:18,991 INFO [master/5a29627a94df:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:47:18,993 INFO [master/5a29627a94df:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T15:47:18,999 INFO [master/5a29627a94df:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-12-05T15:47:19,001 INFO [master/5a29627a94df:0:becomeActiveMaster {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-12-05T15:47:19,011 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.doInit(RegionGroupingProvider.java:156) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.init(AbstractWALProvider.java:126) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:263) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T15:47:19,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741828_1004 (size=1189) 2024-12-05T15:47:19,461 INFO [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store 2024-12-05T15:47:19,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741829_1005 (size=34) 2024-12-05T15:47:19,883 INFO [master/5a29627a94df:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T15:47:19,886 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:19,887 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T15:47:19,887 INFO [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T15:47:19,888 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T15:47:19,889 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T15:47:19,889 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T15:47:19,889 INFO [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T15:47:19,890 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733413639887Disabling compacts and flushes for region at 1733413639887Disabling writes for close at 1733413639889 (+2 ms)Writing region close event to WAL at 1733413639889Closed at 1733413639889 2024-12-05T15:47:19,893 WARN [master/5a29627a94df:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/.initializing 2024-12-05T15:47:19,893 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/WALs/5a29627a94df,33117,1733413636177 2024-12-05T15:47:19,898 INFO [master/5a29627a94df:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T15:47:19,904 INFO [master/5a29627a94df:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 5a29627a94df,33117,1733413636177.regiongroup-0 2024-12-05T15:47:19,921 INFO [master/5a29627a94df:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5a29627a94df%2C33117%2C1733413636177.5a29627a94df%2C33117%2C1733413636177.regiongroup-0, suffix=, logDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/WALs/5a29627a94df,33117,1733413636177, archiveDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/oldWALs, maxLogs=10 2024-12-05T15:47:19,952 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/WALs/5a29627a94df,33117,1733413636177/5a29627a94df%2C33117%2C1733413636177.5a29627a94df%2C33117%2C1733413636177.regiongroup-0.1733413639926, exclude list is [], retry=0 2024-12-05T15:47:19,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:47:19,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-05T15:47:20,008 INFO [master/5a29627a94df:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/WALs/5a29627a94df,33117,1733413636177/5a29627a94df%2C33117%2C1733413636177.5a29627a94df%2C33117%2C1733413636177.regiongroup-0.1733413639926 2024-12-05T15:47:20,009 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:47:20,009 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T15:47:20,010 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:20,014 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T15:47:20,015 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T15:47:20,053 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T15:47:20,076 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T15:47:20,080 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:20,083 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T15:47:20,083 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T15:47:20,087 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T15:47:20,087 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:20,088 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:47:20,088 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T15:47:20,091 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T15:47:20,091 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:20,093 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:47:20,093 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T15:47:20,096 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T15:47:20,096 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:20,097 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:47:20,098 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T15:47:20,101 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T15:47:20,102 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T15:47:20,109 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T15:47:20,109 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T15:47:20,113 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T15:47:20,117 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T15:47:20,121 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T15:47:20,123 INFO [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10266500320, jitterRate=-0.043857648968696594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T15:47:20,129 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733413640030Initializing all the Stores at 1733413640033 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733413640033Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413640035 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413640035Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413640035Cleaning up temporary data from old regions at 1733413640109 (+74 ms)Region opened successfully at 1733413640129 (+20 ms) 2024-12-05T15:47:20,131 INFO [master/5a29627a94df:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T15:47:20,160 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bb6404c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=5a29627a94df/172.17.0.3:0 2024-12-05T15:47:20,191 INFO [master/5a29627a94df:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T15:47:20,203 INFO [master/5a29627a94df:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T15:47:20,203 INFO [master/5a29627a94df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T15:47:20,206 INFO [master/5a29627a94df:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T15:47:20,207 INFO [master/5a29627a94df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T15:47:20,213 INFO [master/5a29627a94df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-05T15:47:20,213 INFO [master/5a29627a94df:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T15:47:20,239 INFO [master/5a29627a94df:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T15:47:20,265 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2024-12-05T15:47:20,320 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/balancer already deleted, retry=false 2024-12-05T15:47:20,323 INFO [master/5a29627a94df:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T15:47:20,324 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2024-12-05T15:47:20,335 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/normalizer already deleted, retry=false 2024-12-05T15:47:20,338 INFO [master/5a29627a94df:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T15:47:20,342 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2024-12-05T15:47:20,352 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/split already deleted, retry=false 2024-12-05T15:47:20,353 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2024-12-05T15:47:20,361 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/merge already deleted, retry=false 2024-12-05T15:47:20,378 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T15:47:20,386 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/snapshot-cleanup already deleted, retry=false 2024-12-05T15:47:20,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-12-05T15:47:20,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-12-05T15:47:20,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:47:20,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:47:20,397 INFO [master/5a29627a94df:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=5a29627a94df,33117,1733413636177, sessionid=0x101a75acab50000, setting cluster-up flag (Was=false) 2024-12-05T15:47:20,404 INFO [master/5a29627a94df:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2024-12-05T15:47:20,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:47:20,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:47:20,453 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2024-12-05T15:47:20,455 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5a29627a94df,33117,1733413636177 2024-12-05T15:47:20,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:47:20,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:47:20,495 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2024-12-05T15:47:20,497 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5a29627a94df,33117,1733413636177 2024-12-05T15:47:20,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:47:20,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:47:20,545 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2024-12-05T15:47:20,547 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5a29627a94df,33117,1733413636177 2024-12-05T15:47:20,552 INFO [master/5a29627a94df:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T15:47:20,619 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(746): ClusterId : 6ce2a395-fcd1-42e7-9dc1-a423aee17fc6 2024-12-05T15:47:20,620 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T15:47:20,621 INFO [RS:0;5a29627a94df:34815 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2024-12-05T15:47:20,622 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T15:47:20,628 INFO [master/5a29627a94df:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T15:47:20,630 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T15:47:20,630 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2024-12-05T15:47:20,634 INFO [master/5a29627a94df:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T15:47:20,638 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2024-12-05T15:47:20,638 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T15:47:20,640 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 5a29627a94df,33117,1733413636177 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T15:47:20,646 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T15:47:20,646 DEBUG [RS:0;5a29627a94df:34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e74c963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=5a29627a94df/172.17.0.3:0 2024-12-05T15:47:20,647 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/5a29627a94df:0, corePoolSize=5, maxPoolSize=5 2024-12-05T15:47:20,647 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/5a29627a94df:0, corePoolSize=5, maxPoolSize=5 2024-12-05T15:47:20,648 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/5a29627a94df:0, corePoolSize=5, maxPoolSize=5 2024-12-05T15:47:20,648 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/5a29627a94df:0, corePoolSize=5, maxPoolSize=5 2024-12-05T15:47:20,648 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/5a29627a94df:0, corePoolSize=10, maxPoolSize=10 2024-12-05T15:47:20,648 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/5a29627a94df:0, corePoolSize=1, maxPoolSize=1 2024-12-05T15:47:20,648 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/5a29627a94df:0, corePoolSize=2, maxPoolSize=2 2024-12-05T15:47:20,649 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/5a29627a94df:0, corePoolSize=1, maxPoolSize=1 2024-12-05T15:47:20,654 INFO [master/5a29627a94df:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733413670653 2024-12-05T15:47:20,654 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T15:47:20,655 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T15:47:20,655 INFO [master/5a29627a94df:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T15:47:20,657 INFO [master/5a29627a94df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T15:47:20,661 INFO [master/5a29627a94df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T15:47:20,661 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:20,662 INFO [master/5a29627a94df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2024-12-05T15:47:20,661 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T15:47:20,662 INFO [master/5a29627a94df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T15:47:20,662 INFO [master/5a29627a94df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T15:47:20,662 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;5a29627a94df:34815 2024-12-05T15:47:20,662 INFO [master/5a29627a94df:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T15:47:20,663 INFO [master/5a29627a94df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:20,667 INFO [master/5a29627a94df:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T15:47:20,667 INFO [RS:0;5a29627a94df:34815 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T15:47:20,668 INFO [RS:0;5a29627a94df:34815 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T15:47:20,668 INFO [master/5a29627a94df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T15:47:20,668 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T15:47:20,668 INFO [master/5a29627a94df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T15:47:20,671 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(2659): reportForDuty to master=5a29627a94df,33117,1733413636177 with port=34815, startcode=1733413637643 2024-12-05T15:47:20,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741831_1007 (size=1321) 2024-12-05T15:47:20,680 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7afeb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:20,680 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:47:20,684 DEBUG [RS:0;5a29627a94df:34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T15:47:20,684 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:47:20,708 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:47:20,716 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42433, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T15:47:20,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:47:20,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:47:20,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55aa077c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:20,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:47:20,722 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33117 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T15:47:20,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:47:20,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:20,727 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:47:20,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c90f8c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:20,731 INFO [master/5a29627a94df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-12-05T15:47:20,733 INFO [master/5a29627a94df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T15:47:20,733 INFO [master/5a29627a94df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T15:47:20,735 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/5a29627a94df:0:becomeActiveMaster-HFileCleaner.large.0-1733413640735,5,FailOnTimeoutGroup] 2024-12-05T15:47:20,736 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/5a29627a94df:0:becomeActiveMaster-HFileCleaner.small.0-1733413640736,5,FailOnTimeoutGroup] 2024-12-05T15:47:20,736 INFO [master/5a29627a94df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:20,737 INFO [master/5a29627a94df:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T15:47:20,738 INFO [master/5a29627a94df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:20,739 INFO [master/5a29627a94df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:20,741 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-05T15:47:20,741 WARN [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-05T15:47:20,843 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(2659): reportForDuty to master=5a29627a94df,33117,1733413636177 with port=34815, startcode=1733413637643 2024-12-05T15:47:20,845 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33117 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5a29627a94df,34815,1733413637643 2024-12-05T15:47:20,847 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33117 {}] master.ServerManager(517): Registering regionserver=5a29627a94df,34815,1733413637643 2024-12-05T15:47:20,856 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:47:20,856 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37233 2024-12-05T15:47:20,857 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=39857 2024-12-05T15:47:20,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-12-05T15:47:20,891 DEBUG [RS:0;5a29627a94df:34815 {}] zookeeper.ZKUtil(111): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on existing znode=/1/rs/5a29627a94df,34815,1733413637643 2024-12-05T15:47:20,892 WARN [RS:0;5a29627a94df:34815 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T15:47:20,892 INFO [RS:0;5a29627a94df:34815 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-12-05T15:47:20,892 INFO [RS:0;5a29627a94df:34815 {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-12-05T15:47:20,892 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643 2024-12-05T15:47:20,894 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5a29627a94df,34815,1733413637643] 2024-12-05T15:47:20,916 INFO [RS:0;5a29627a94df:34815 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T15:47:20,927 INFO [RS:0;5a29627a94df:34815 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T15:47:20,932 INFO [RS:0;5a29627a94df:34815 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T15:47:20,932 INFO [RS:0;5a29627a94df:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:20,933 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T15:47:20,939 INFO [RS:0;5a29627a94df:34815 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T15:47:20,940 INFO [RS:0;5a29627a94df:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:20,941 DEBUG [RS:0;5a29627a94df:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5a29627a94df:0, corePoolSize=1, maxPoolSize=1 2024-12-05T15:47:20,941 DEBUG [RS:0;5a29627a94df:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5a29627a94df:0, corePoolSize=1, maxPoolSize=1 2024-12-05T15:47:20,941 DEBUG [RS:0;5a29627a94df:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5a29627a94df:0, corePoolSize=1, maxPoolSize=1 2024-12-05T15:47:20,941 DEBUG [RS:0;5a29627a94df:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5a29627a94df:0, corePoolSize=1, maxPoolSize=1 2024-12-05T15:47:20,941 DEBUG [RS:0;5a29627a94df:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5a29627a94df:0, corePoolSize=1, maxPoolSize=1 2024-12-05T15:47:20,941 DEBUG [RS:0;5a29627a94df:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5a29627a94df:0, corePoolSize=2, maxPoolSize=2 2024-12-05T15:47:20,941 DEBUG [RS:0;5a29627a94df:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5a29627a94df:0, corePoolSize=1, maxPoolSize=1 2024-12-05T15:47:20,942 DEBUG [RS:0;5a29627a94df:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5a29627a94df:0, corePoolSize=1, maxPoolSize=1 2024-12-05T15:47:20,942 DEBUG [RS:0;5a29627a94df:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5a29627a94df:0, corePoolSize=1, maxPoolSize=1 2024-12-05T15:47:20,942 DEBUG [RS:0;5a29627a94df:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5a29627a94df:0, corePoolSize=1, maxPoolSize=1 2024-12-05T15:47:20,942 DEBUG [RS:0;5a29627a94df:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5a29627a94df:0, corePoolSize=1, maxPoolSize=1 2024-12-05T15:47:20,942 DEBUG [RS:0;5a29627a94df:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5a29627a94df:0, corePoolSize=1, maxPoolSize=1 2024-12-05T15:47:20,942 DEBUG [RS:0;5a29627a94df:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0, corePoolSize=3, maxPoolSize=3 2024-12-05T15:47:20,942 DEBUG [RS:0;5a29627a94df:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5a29627a94df:0, corePoolSize=3, maxPoolSize=3 2024-12-05T15:47:20,943 INFO [RS:0;5a29627a94df:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:20,943 INFO [RS:0;5a29627a94df:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:20,943 INFO [RS:0;5a29627a94df:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:20,944 INFO [RS:0;5a29627a94df:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:20,944 INFO [RS:0;5a29627a94df:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:20,944 INFO [RS:0;5a29627a94df:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=5a29627a94df,34815,1733413637643-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T15:47:20,959 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T15:47:20,961 INFO [RS:0;5a29627a94df:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=5a29627a94df,34815,1733413637643-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:20,961 INFO [RS:0;5a29627a94df:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:20,961 INFO [RS:0;5a29627a94df:34815 {}] regionserver.Replication(171): 5a29627a94df,34815,1733413637643 started 2024-12-05T15:47:20,976 INFO [RS:0;5a29627a94df:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:20,976 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1482): Serving as 5a29627a94df,34815,1733413637643, RpcServer on 5a29627a94df/172.17.0.3:34815, sessionid=0x101a75acab50001 2024-12-05T15:47:20,977 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T15:47:20,977 DEBUG [RS:0;5a29627a94df:34815 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5a29627a94df,34815,1733413637643 2024-12-05T15:47:20,977 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5a29627a94df,34815,1733413637643' 2024-12-05T15:47:20,977 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2024-12-05T15:47:20,978 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2024-12-05T15:47:20,979 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T15:47:20,979 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2024-12-05T15:47:20,979 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5a29627a94df,34815,1733413637643' 2024-12-05T15:47:20,979 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-05T15:47:20,979 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-05T15:47:20,980 INFO [RS:0;5a29627a94df:34815 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2024-12-05T15:47:20,980 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2024-12-05T15:47:20,980 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T15:47:20,980 DEBUG [RS:0;5a29627a94df:34815 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5a29627a94df,34815,1733413637643 2024-12-05T15:47:20,980 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5a29627a94df,34815,1733413637643' 2024-12-05T15:47:20,980 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2024-12-05T15:47:20,981 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2024-12-05T15:47:20,981 DEBUG [RS:0;5a29627a94df:34815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T15:47:20,982 INFO [RS:0;5a29627a94df:34815 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T15:47:20,982 INFO [RS:0;5a29627a94df:34815 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T15:47:21,074 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T15:47:21,075 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:47:21,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741832_1008 (size=32) 2024-12-05T15:47:21,086 INFO [RS:0;5a29627a94df:34815 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T15:47:21,086 INFO [RS:0;5a29627a94df:34815 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 5a29627a94df,34815,1733413637643.regiongroup-0 2024-12-05T15:47:21,089 INFO [RS:0;5a29627a94df:34815 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0, suffix=, logDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643, archiveDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs, maxLogs=32 2024-12-05T15:47:21,102 DEBUG [RS:0;5a29627a94df:34815 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413641091, exclude list is [], retry=0 2024-12-05T15:47:21,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:47:21,110 INFO [RS:0;5a29627a94df:34815 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413641091 2024-12-05T15:47:21,111 DEBUG [RS:0;5a29627a94df:34815 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:47:21,486 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:21,488 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T15:47:21,491 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T15:47:21,491 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:21,492 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T15:47:21,492 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T15:47:21,495 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T15:47:21,495 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:21,496 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T15:47:21,496 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T15:47:21,498 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T15:47:21,498 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:21,499 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T15:47:21,500 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T15:47:21,502 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T15:47:21,502 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:21,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T15:47:21,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T15:47:21,505 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740 2024-12-05T15:47:21,506 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740 2024-12-05T15:47:21,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T15:47:21,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T15:47:21,510 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T15:47:21,512 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T15:47:21,517 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T15:47:21,518 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10492090560, jitterRate=-0.022847920656204224}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T15:47:21,521 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733413641486Initializing all the Stores at 1733413641488 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733413641488Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733413641488Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413641488Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733413641488Cleaning up temporary data from old regions at 1733413641509 (+21 ms)Region opened successfully at 1733413641520 (+11 ms) 2024-12-05T15:47:21,521 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T15:47:21,521 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T15:47:21,521 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T15:47:21,521 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T15:47:21,521 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T15:47:21,522 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T15:47:21,522 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733413641521Disabling compacts and flushes for region at 1733413641521Disabling writes for close at 1733413641521Writing region close event to WAL at 1733413641522 (+1 ms)Closed at 1733413641522 2024-12-05T15:47:21,526 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T15:47:21,526 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T15:47:21,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T15:47:21,540 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T15:47:21,543 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T15:47:21,695 DEBUG [5a29627a94df:33117 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T15:47:21,709 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:21,716 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5a29627a94df,34815,1733413637643, state=OPENING 2024-12-05T15:47:21,778 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T15:47:21,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:47:21,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:47:21,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-05T15:47:21,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-05T15:47:21,789 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T15:47:21,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:47:21,965 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T15:47:21,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33127, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T15:47:21,978 INFO [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T15:47:21,979 INFO [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-12-05T15:47:21,979 INFO [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-12-05T15:47:21,979 INFO [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T15:47:21,980 INFO [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T15:47:21,983 INFO [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5a29627a94df%2C34815%2C1733413637643.meta, suffix=.meta, logDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643, archiveDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs, maxLogs=32 2024-12-05T15:47:21,995 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413641984.meta, exclude list is [], retry=0 2024-12-05T15:47:21,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:47:22,002 INFO [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413641984.meta 2024-12-05T15:47:22,002 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:47:22,003 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T15:47:22,007 INFO [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:47:22,008 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T15:47:22,010 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T15:47:22,011 INFO [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T15:47:22,014 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T15:47:22,014 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:22,015 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T15:47:22,015 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T15:47:22,018 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T15:47:22,019 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T15:47:22,019 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:22,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T15:47:22,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T15:47:22,022 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T15:47:22,022 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:22,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T15:47:22,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T15:47:22,024 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T15:47:22,024 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:22,025 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T15:47:22,025 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T15:47:22,026 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T15:47:22,026 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:22,027 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T15:47:22,027 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T15:47:22,029 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740 2024-12-05T15:47:22,032 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740 2024-12-05T15:47:22,034 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T15:47:22,034 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T15:47:22,035 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T15:47:22,038 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T15:47:22,040 INFO [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10955972640, jitterRate=0.020354464650154114}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T15:47:22,040 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T15:47:22,042 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733413642015Writing region info on filesystem at 1733413642015Initializing all the Stores at 1733413642017 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733413642017Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733413642017Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413642017Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733413642017Cleaning up temporary data from old regions at 1733413642034 (+17 ms)Running coprocessor post-open hooks at 1733413642040 (+6 ms)Region opened successfully at 1733413642042 (+2 ms) 2024-12-05T15:47:22,050 INFO [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733413641958 2024-12-05T15:47:22,062 DEBUG [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T15:47:22,063 INFO [RS_OPEN_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T15:47:22,064 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:22,066 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5a29627a94df,34815,1733413637643, state=OPEN 2024-12-05T15:47:22,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-12-05T15:47:22,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-12-05T15:47:22,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-05T15:47:22,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-05T15:47:22,077 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5a29627a94df,34815,1733413637643 2024-12-05T15:47:22,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T15:47:22,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5a29627a94df,34815,1733413637643 in 286 msec 2024-12-05T15:47:22,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T15:47:22,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 551 msec 2024-12-05T15:47:22,090 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T15:47:22,090 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T15:47:22,106 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:47:22,107 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:47:22,124 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:47:22,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37287, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:47:22,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5680 sec 2024-12-05T15:47:22,149 INFO [master/5a29627a94df:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733413642149, completionTime=-1 2024-12-05T15:47:22,152 INFO [master/5a29627a94df:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T15:47:22,152 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T15:47:22,186 INFO [master/5a29627a94df:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-05T15:47:22,186 INFO [master/5a29627a94df:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733413702186 2024-12-05T15:47:22,186 INFO [master/5a29627a94df:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733413762186 2024-12-05T15:47:22,186 INFO [master/5a29627a94df:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-12-05T15:47:22,189 INFO [master/5a29627a94df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5a29627a94df,33117,1733413636177-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:22,190 INFO [master/5a29627a94df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5a29627a94df,33117,1733413636177-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:22,190 INFO [master/5a29627a94df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5a29627a94df,33117,1733413636177-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:22,192 INFO [master/5a29627a94df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-5a29627a94df:33117, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:22,193 INFO [master/5a29627a94df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:22,193 INFO [master/5a29627a94df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:22,198 DEBUG [master/5a29627a94df:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T15:47:22,232 INFO [master/5a29627a94df:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.352sec 2024-12-05T15:47:22,233 INFO [master/5a29627a94df:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T15:47:22,235 INFO [master/5a29627a94df:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T15:47:22,237 INFO [master/5a29627a94df:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T15:47:22,238 INFO [master/5a29627a94df:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T15:47:22,238 INFO [master/5a29627a94df:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T15:47:22,239 INFO [master/5a29627a94df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5a29627a94df,33117,1733413636177-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T15:47:22,240 INFO [master/5a29627a94df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5a29627a94df,33117,1733413636177-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T15:47:22,251 DEBUG [master/5a29627a94df:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T15:47:22,253 INFO [master/5a29627a94df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T15:47:22,253 INFO [master/5a29627a94df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5a29627a94df,33117,1733413636177-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T15:47:22,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54190a72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:22,324 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T15:47:22,324 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T15:47:22,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:47:22,327 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:47:22,335 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:47:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:47:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:47:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fc2f415, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:47:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:47:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:22,340 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:47:22,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55de5b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:22,343 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:47:22,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:47:22,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:47:22,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:47:22,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=5a29627a94df,33117,1733413636177 2024-12-05T15:47:22,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2305): Starting mini mapreduce cluster... 2024-12-05T15:47:22,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/test.cache.data in system properties and HBase conf 2024-12-05T15:47:22,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T15:47:22,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir in system properties and HBase conf 2024-12-05T15:47:22,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T15:47:22,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T15:47:22,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-05T15:47:22,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T15:47:22,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T15:47:22,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T15:47:22,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T15:47:22,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T15:47:22,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T15:47:22,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T15:47:22,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T15:47:22,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T15:47:22,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/nfs.dump.dir in system properties and HBase conf 2024-12-05T15:47:22,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/java.io.tmpdir in system properties and HBase conf 2024-12-05T15:47:22,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T15:47:22,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T15:47:22,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T15:47:22,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741835_1011 (size=592039) 2024-12-05T15:47:22,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741836_1012 (size=1663647) 2024-12-05T15:47:24,260 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T15:47:24,332 WARN [Thread-251 {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T15:47:24,519 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T15:47:24,519 WARN [Thread-251 {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-12-05T15:47:24,520 INFO [Thread-251 {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T15:47:24,520 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T15:47:24,520 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T15:47:24,520 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T15:47:24,521 INFO [Thread-251 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T15:47:24,521 INFO [Thread-251 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T15:47:24,521 INFO [Thread-251 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T15:47:24,521 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a344172{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,AVAILABLE} 2024-12-05T15:47:24,522 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71ff468e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-05T15:47:24,522 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T15:47:24,531 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ef40a45{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,AVAILABLE} 2024-12-05T15:47:24,532 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336bed82{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-05T15:47:24,638 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2024-12-05T15:47:24,638 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2024-12-05T15:47:24,638 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-12-05T15:47:24,640 INFO [Thread-251 {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-12-05T15:47:24,683 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-05T15:47:24,722 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-05T15:47:24,911 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2024-12-05T15:47:24,931 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2d6554c9{jobhistory,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/java.io.tmpdir/jetty-localhost-40841-hadoop-yarn-common-3_4_1_jar-_-any-5026841807514250320/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-12-05T15:47:24,931 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67e7621b{cluster,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/java.io.tmpdir/jetty-localhost-43907-hadoop-yarn-common-3_4_1_jar-_-any-17655852774794934666/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-12-05T15:47:24,932 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3f7727e1{HTTP/1.1, (http/1.1)}{localhost:43907} 2024-12-05T15:47:24,932 INFO [Thread-251 {}] server.AbstractConnector(333): Started ServerConnector@34d2c2ee{HTTP/1.1, (http/1.1)}{localhost:40841} 2024-12-05T15:47:24,932 INFO [Time-limited test {}] server.Server(415): Started @14422ms 2024-12-05T15:47:24,932 INFO [Thread-251 {}] server.Server(415): Started @14422ms 2024-12-05T15:47:25,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741837_1013 (size=5) 2024-12-05T15:47:25,823 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-12-05T15:47:25,827 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T15:47:25,860 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-12-05T15:47:25,861 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T15:47:25,862 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T15:47:25,862 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T15:47:25,863 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T15:47:25,864 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T15:47:25,864 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a9ede50{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,AVAILABLE} 2024-12-05T15:47:25,865 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@591f9edc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-05T15:47:25,934 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-12-05T15:47:25,935 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-12-05T15:47:25,935 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-12-05T15:47:25,935 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-12-05T15:47:25,946 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-05T15:47:25,955 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-05T15:47:26,087 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-05T15:47:26,096 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@30e0ce95{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/java.io.tmpdir/jetty-localhost-36083-hadoop-yarn-common-3_4_1_jar-_-any-12406963436869054849/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-05T15:47:26,096 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5668db5e{HTTP/1.1, (http/1.1)}{localhost:36083} 2024-12-05T15:47:26,096 INFO [Time-limited test {}] server.Server(415): Started @15586ms 2024-12-05T15:47:26,220 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-12-05T15:47:26,222 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T15:47:26,232 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-12-05T15:47:26,233 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T15:47:26,236 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T15:47:26,236 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T15:47:26,236 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T15:47:26,237 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T15:47:26,237 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31dc727{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,AVAILABLE} 2024-12-05T15:47:26,238 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78ebccb1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-05T15:47:26,279 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-12-05T15:47:26,279 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-12-05T15:47:26,279 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-12-05T15:47:26,279 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-12-05T15:47:26,286 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-05T15:47:26,290 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-05T15:47:26,353 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-05T15:47:26,359 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@333abf77{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/java.io.tmpdir/jetty-localhost-42249-hadoop-yarn-common-3_4_1_jar-_-any-7641494719087353552/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-05T15:47:26,360 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3ede3ae6{HTTP/1.1, (http/1.1)}{localhost:42249} 2024-12-05T15:47:26,360 INFO [Time-limited test {}] server.Server(415): Started @15850ms 2024-12-05T15:47:26,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2341): Mini mapreduce cluster started 2024-12-05T15:47:26,382 INFO [Time-limited test {}] backup.TestBackupBase(336): ROOTDIR hdfs://localhost:37233/backupUT 2024-12-05T15:47:26,386 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:47:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:47:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3a28ad61 2024-12-05T15:47:26,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:47:26,397 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:47:26,400 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns1'} 2024-12-05T15:47:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns1 2024-12-05T15:47:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T15:47:26,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns1 in 20 msec 2024-12-05T15:47:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T15:47:26,528 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns1 completed 2024-12-05T15:47:26,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns2'} 2024-12-05T15:47:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns2 2024-12-05T15:47:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-12-05T15:47:26,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns2 in 14 msec 2024-12-05T15:47:26,572 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T15:47:26,573 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-05T15:47:26,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:47:26,581 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2024-12-05T15:47:26,582 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T15:47:26,582 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-05T15:47:26,582 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T15:47:26,582 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-05T15:47:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-12-05T15:47:26,647 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns2 completed 2024-12-05T15:47:26,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns3'} 2024-12-05T15:47:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=6, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns3 2024-12-05T15:47:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-12-05T15:47:26,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns3 in 13 msec 2024-12-05T15:47:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-12-05T15:47:26,757 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns3 completed 2024-12-05T15:47:26,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns4'} 2024-12-05T15:47:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns4 2024-12-05T15:47:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T15:47:26,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns4 in 13 msec 2024-12-05T15:47:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T15:47:26,868 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns4 completed 2024-12-05T15:47:26,876 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test-1733413646382', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:47:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test-1733413646382 2024-12-05T15:47:26,885 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test-1733413646382 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T15:47:26,887 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test-1733413646382" procId is: 8 2024-12-05T15:47:26,887 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:26,889 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test-1733413646382 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T15:47:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-05T15:47:26,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741838_1014 (size=367) 2024-12-05T15:47:26,911 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T15:47:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-05T15:47:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-05T15:47:27,303 INFO [RegionOpenAndInit-test-1733413646382-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 24a988b9e3812c25fcebd4f4072c48f0, NAME => 'test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1733413646382', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:47:27,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741839_1015 (size=53) 2024-12-05T15:47:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-05T15:47:27,535 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-12-05T15:47:27,714 DEBUG [RegionOpenAndInit-test-1733413646382-pool-0 {}] regionserver.HRegion(898): Instantiated test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:27,714 DEBUG [RegionOpenAndInit-test-1733413646382-pool-0 {}] regionserver.HRegion(1722): Closing 24a988b9e3812c25fcebd4f4072c48f0, disabling compactions & flushes 2024-12-05T15:47:27,714 INFO [RegionOpenAndInit-test-1733413646382-pool-0 {}] regionserver.HRegion(1755): Closing region test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:27,714 DEBUG [RegionOpenAndInit-test-1733413646382-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:27,714 DEBUG [RegionOpenAndInit-test-1733413646382-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. after waiting 0 ms 2024-12-05T15:47:27,714 DEBUG [RegionOpenAndInit-test-1733413646382-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:27,714 INFO [RegionOpenAndInit-test-1733413646382-pool-0 {}] regionserver.HRegion(1973): Closed test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:27,714 DEBUG [RegionOpenAndInit-test-1733413646382-pool-0 {}] regionserver.HRegion(1676): Region close journal for 24a988b9e3812c25fcebd4f4072c48f0: Waiting for close lock at 1733413647714Disabling compacts and flushes for region at 1733413647714Disabling writes for close at 1733413647714Writing region close event to WAL at 1733413647714Closed at 1733413647714 2024-12-05T15:47:27,717 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test-1733413646382 execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T15:47:27,722 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733413647717"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733413647717"}]},"ts":"1733413647717"} 2024-12-05T15:47:27,726 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T15:47:27,728 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test-1733413646382 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T15:47:27,732 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1733413646382","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413647729"}]},"ts":"1733413647729"} 2024-12-05T15:47:27,736 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1733413646382, state=ENABLING in hbase:meta 2024-12-05T15:47:27,738 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, ASSIGN}] 2024-12-05T15:47:27,740 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, ASSIGN 2024-12-05T15:47:27,742 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, ASSIGN; state=OFFLINE, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=false 2024-12-05T15:47:27,893 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=24a988b9e3812c25fcebd4f4072c48f0, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:27,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, ASSIGN because future has completed 2024-12-05T15:47:27,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:47:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-05T15:47:28,072 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(132): Open test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:28,072 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T15:47:28,072 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] monitor.StreamSlowMonitor(122): New stream slow monitor 5a29627a94df,34815,1733413637643.regiongroup-1 2024-12-05T15:47:28,075 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1, suffix=, logDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643, archiveDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs, maxLogs=32 2024-12-05T15:47:28,089 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413648077, exclude list is [], retry=0 2024-12-05T15:47:28,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:47:28,096 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413648077 2024-12-05T15:47:28,097 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:47:28,097 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7752): Opening region: {ENCODED => 24a988b9e3812c25fcebd4f4072c48f0, NAME => 'test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0.', STARTKEY => '', ENDKEY => ''} 2024-12-05T15:47:28,098 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:47:28,098 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1733413646382 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:28,098 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(898): Instantiated test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:28,098 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7794): checking encryption for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:28,098 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7797): checking classloading for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:28,101 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:28,103 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24a988b9e3812c25fcebd4f4072c48f0 columnFamilyName f 2024-12-05T15:47:28,104 DEBUG [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:28,105 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.HStore(327): Store=24a988b9e3812c25fcebd4f4072c48f0/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:47:28,105 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1038): replaying wal for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:28,106 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:28,107 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:28,108 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1048): stopping wal replay for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:28,108 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1060): Cleaning up temporary data for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:28,111 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1093): writing seq id for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:28,115 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T15:47:28,116 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1114): Opened 24a988b9e3812c25fcebd4f4072c48f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12020498240, jitterRate=0.11949613690376282}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T15:47:28,116 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:28,117 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1006): Region open journal for 24a988b9e3812c25fcebd4f4072c48f0: Running coprocessor pre-open hook at 1733413648098Writing region info on filesystem at 1733413648098Initializing all the Stores at 1733413648100 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413648100Cleaning up temporary data from old regions at 1733413648108 (+8 ms)Running coprocessor post-open hooks at 1733413648116 (+8 ms)Region opened successfully at 1733413648117 (+1 ms) 2024-12-05T15:47:28,119 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0., pid=10, masterSystemTime=1733413648052 2024-12-05T15:47:28,123 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:28,123 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(153): Opened test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:28,124 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=24a988b9e3812c25fcebd4f4072c48f0, regionState=OPEN, openSeqNum=2, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:28,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:47:28,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-05T15:47:28,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0, server=5a29627a94df,34815,1733413637643 in 232 msec 2024-12-05T15:47:28,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-05T15:47:28,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, ASSIGN in 398 msec 2024-12-05T15:47:28,143 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test-1733413646382 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T15:47:28,143 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1733413646382","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413648143"}]},"ts":"1733413648143"} 2024-12-05T15:47:28,147 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1733413646382, state=ENABLED in hbase:meta 2024-12-05T15:47:28,149 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test-1733413646382 execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T15:47:28,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; CreateTableProcedure table=test-1733413646382 in 1.2710 sec 2024-12-05T15:47:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-05T15:47:29,038 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test-1733413646382 completed 2024-12-05T15:47:29,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@449bd6dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:29,039 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:47:29,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:47:29,041 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:47:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:47:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:47:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@717cfedb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:47:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:47:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:29,044 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:47:29,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dc39053, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:29,049 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:47:29,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:47:29,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:47:29,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:47:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733413646382', row='row0', locateType=CURRENT is [region=test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:47:29,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HRegion(8528): writing data to region test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. with WAL disabled. Data may be lost in the event of a crash. 2024-12-05T15:47:29,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns2:test-17334136463821', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:47:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:test-17334136463821 2024-12-05T15:47:29,279 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17334136463821 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T15:47:29,279 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:29,279 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns2" qualifier: "test-17334136463821" procId is: 11 2024-12-05T15:47:29,280 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:test-17334136463821 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T15:47:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T15:47:29,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741841_1017 (size=364) 2024-12-05T15:47:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T15:47:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T15:47:29,692 INFO [RegionOpenAndInit-ns2:test-17334136463821-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b7877185d3b50523892995990a3cd728, NAME => 'ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17334136463821', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:47:29,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741842_1018 (size=50) 2024-12-05T15:47:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T15:47:30,101 DEBUG [RegionOpenAndInit-ns2:test-17334136463821-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:30,101 DEBUG [RegionOpenAndInit-ns2:test-17334136463821-pool-0 {}] regionserver.HRegion(1722): Closing b7877185d3b50523892995990a3cd728, disabling compactions & flushes 2024-12-05T15:47:30,101 INFO [RegionOpenAndInit-ns2:test-17334136463821-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:47:30,101 DEBUG [RegionOpenAndInit-ns2:test-17334136463821-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:47:30,101 DEBUG [RegionOpenAndInit-ns2:test-17334136463821-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. after waiting 0 ms 2024-12-05T15:47:30,101 DEBUG [RegionOpenAndInit-ns2:test-17334136463821-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:47:30,101 INFO [RegionOpenAndInit-ns2:test-17334136463821-pool-0 {}] regionserver.HRegion(1973): Closed ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:47:30,101 DEBUG [RegionOpenAndInit-ns2:test-17334136463821-pool-0 {}] regionserver.HRegion(1676): Region close journal for b7877185d3b50523892995990a3cd728: Waiting for close lock at 1733413650101Disabling compacts and flushes for region at 1733413650101Disabling writes for close at 1733413650101Writing region close event to WAL at 1733413650101Closed at 1733413650101 2024-12-05T15:47:30,103 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:test-17334136463821 execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T15:47:30,104 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1733413650103"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733413650103"}]},"ts":"1733413650103"} 2024-12-05T15:47:30,106 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T15:47:30,108 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:test-17334136463821 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T15:47:30,108 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17334136463821","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413650108"}]},"ts":"1733413650108"} 2024-12-05T15:47:30,110 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17334136463821, state=ENABLING in hbase:meta 2024-12-05T15:47:30,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17334136463821, region=b7877185d3b50523892995990a3cd728, ASSIGN}] 2024-12-05T15:47:30,112 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17334136463821, region=b7877185d3b50523892995990a3cd728, ASSIGN 2024-12-05T15:47:30,114 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:test-17334136463821, region=b7877185d3b50523892995990a3cd728, ASSIGN; state=OFFLINE, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=false 2024-12-05T15:47:30,264 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=b7877185d3b50523892995990a3cd728, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:30,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:test-17334136463821, region=b7877185d3b50523892995990a3cd728, ASSIGN because future has completed 2024-12-05T15:47:30,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7877185d3b50523892995990a3cd728, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:47:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T15:47:30,427 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:47:30,428 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => b7877185d3b50523892995990a3cd728, NAME => 'ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728.', STARTKEY => '', ENDKEY => ''} 2024-12-05T15:47:30,428 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:47:30,428 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17334136463821 b7877185d3b50523892995990a3cd728 2024-12-05T15:47:30,428 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:30,429 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for b7877185d3b50523892995990a3cd728 2024-12-05T15:47:30,429 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for b7877185d3b50523892995990a3cd728 2024-12-05T15:47:30,431 INFO [StoreOpener-b7877185d3b50523892995990a3cd728-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region b7877185d3b50523892995990a3cd728 2024-12-05T15:47:30,433 INFO [StoreOpener-b7877185d3b50523892995990a3cd728-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7877185d3b50523892995990a3cd728 columnFamilyName f 2024-12-05T15:47:30,434 DEBUG [StoreOpener-b7877185d3b50523892995990a3cd728-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:30,434 INFO [StoreOpener-b7877185d3b50523892995990a3cd728-1 {}] regionserver.HStore(327): Store=b7877185d3b50523892995990a3cd728/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:47:30,435 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for b7877185d3b50523892995990a3cd728 2024-12-05T15:47:30,436 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728 2024-12-05T15:47:30,437 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728 2024-12-05T15:47:30,437 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for b7877185d3b50523892995990a3cd728 2024-12-05T15:47:30,437 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for b7877185d3b50523892995990a3cd728 2024-12-05T15:47:30,440 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for b7877185d3b50523892995990a3cd728 2024-12-05T15:47:30,443 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T15:47:30,444 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened b7877185d3b50523892995990a3cd728; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10309495200, jitterRate=-0.03985343873500824}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T15:47:30,444 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7877185d3b50523892995990a3cd728 2024-12-05T15:47:30,445 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for b7877185d3b50523892995990a3cd728: Running coprocessor pre-open hook at 1733413650429Writing region info on filesystem at 1733413650429Initializing all the Stores at 1733413650431 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413650431Cleaning up temporary data from old regions at 1733413650437 (+6 ms)Running coprocessor post-open hooks at 1733413650444 (+7 ms)Region opened successfully at 1733413650445 (+1 ms) 2024-12-05T15:47:30,446 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728., pid=13, masterSystemTime=1733413650422 2024-12-05T15:47:30,449 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:47:30,449 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:47:30,450 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=b7877185d3b50523892995990a3cd728, regionState=OPEN, openSeqNum=2, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:30,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7877185d3b50523892995990a3cd728, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:47:30,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-12-05T15:47:30,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure b7877185d3b50523892995990a3cd728, server=5a29627a94df,34815,1733413637643 in 187 msec 2024-12-05T15:47:30,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-05T15:47:30,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:test-17334136463821, region=b7877185d3b50523892995990a3cd728, ASSIGN in 348 msec 2024-12-05T15:47:30,465 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:test-17334136463821 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T15:47:30,466 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17334136463821","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413650465"}]},"ts":"1733413650465"} 2024-12-05T15:47:30,469 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17334136463821, state=ENABLED in hbase:meta 2024-12-05T15:47:30,470 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17334136463821 execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T15:47:30,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:test-17334136463821 in 1.1950 sec 2024-12-05T15:47:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T15:47:31,427 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:test-17334136463821 completed 2024-12-05T15:47:31,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17334136463821', row='row0', locateType=CURRENT is [region=ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:47:31,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HRegion(8528): writing data to region ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. with WAL disabled. Data may be lost in the event of a crash. 2024-12-05T15:47:31,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns3:test-17334136463822', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:47:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns3:test-17334136463822 2024-12-05T15:47:31,598 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17334136463822 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T15:47:31,598 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:31,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns3" qualifier: "test-17334136463822" procId is: 14 2024-12-05T15:47:31,599 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns3:test-17334136463822 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T15:47:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-05T15:47:31,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741843_1019 (size=390) 2024-12-05T15:47:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-05T15:47:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-05T15:47:32,011 INFO [RegionOpenAndInit-ns3:test-17334136463822-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 750a447b5bb19a7405a6a02a816d1329, NAME => 'ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17334136463822', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:47:32,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741844_1020 (size=50) 2024-12-05T15:47:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-05T15:47:32,422 DEBUG [RegionOpenAndInit-ns3:test-17334136463822-pool-0 {}] regionserver.HRegion(898): Instantiated ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:32,422 DEBUG [RegionOpenAndInit-ns3:test-17334136463822-pool-0 {}] regionserver.HRegion(1722): Closing 750a447b5bb19a7405a6a02a816d1329, disabling compactions & flushes 2024-12-05T15:47:32,422 INFO [RegionOpenAndInit-ns3:test-17334136463822-pool-0 {}] regionserver.HRegion(1755): Closing region ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. 2024-12-05T15:47:32,422 DEBUG [RegionOpenAndInit-ns3:test-17334136463822-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. 2024-12-05T15:47:32,423 DEBUG [RegionOpenAndInit-ns3:test-17334136463822-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. after waiting 0 ms 2024-12-05T15:47:32,423 DEBUG [RegionOpenAndInit-ns3:test-17334136463822-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. 2024-12-05T15:47:32,423 INFO [RegionOpenAndInit-ns3:test-17334136463822-pool-0 {}] regionserver.HRegion(1973): Closed ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. 2024-12-05T15:47:32,423 DEBUG [RegionOpenAndInit-ns3:test-17334136463822-pool-0 {}] regionserver.HRegion(1676): Region close journal for 750a447b5bb19a7405a6a02a816d1329: Waiting for close lock at 1733413652422Disabling compacts and flushes for region at 1733413652422Disabling writes for close at 1733413652423 (+1 ms)Writing region close event to WAL at 1733413652423Closed at 1733413652423 2024-12-05T15:47:32,425 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns3:test-17334136463822 execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T15:47:32,425 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1733413652425"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733413652425"}]},"ts":"1733413652425"} 2024-12-05T15:47:32,429 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T15:47:32,431 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns3:test-17334136463822 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T15:47:32,431 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17334136463822","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413652431"}]},"ts":"1733413652431"} 2024-12-05T15:47:32,434 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17334136463822, state=ENABLING in hbase:meta 2024-12-05T15:47:32,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17334136463822, region=750a447b5bb19a7405a6a02a816d1329, ASSIGN}] 2024-12-05T15:47:32,437 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17334136463822, region=750a447b5bb19a7405a6a02a816d1329, ASSIGN 2024-12-05T15:47:32,438 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns3:test-17334136463822, region=750a447b5bb19a7405a6a02a816d1329, ASSIGN; state=OFFLINE, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=false 2024-12-05T15:47:32,589 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=750a447b5bb19a7405a6a02a816d1329, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:32,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns3:test-17334136463822, region=750a447b5bb19a7405a6a02a816d1329, ASSIGN because future has completed 2024-12-05T15:47:32,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 750a447b5bb19a7405a6a02a816d1329, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:47:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-05T15:47:32,753 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. 2024-12-05T15:47:32,754 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => 750a447b5bb19a7405a6a02a816d1329, NAME => 'ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329.', STARTKEY => '', ENDKEY => ''} 2024-12-05T15:47:32,754 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:47:32,754 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17334136463822 750a447b5bb19a7405a6a02a816d1329 2024-12-05T15:47:32,755 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:32,755 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for 750a447b5bb19a7405a6a02a816d1329 2024-12-05T15:47:32,755 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for 750a447b5bb19a7405a6a02a816d1329 2024-12-05T15:47:32,758 INFO [StoreOpener-750a447b5bb19a7405a6a02a816d1329-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 750a447b5bb19a7405a6a02a816d1329 2024-12-05T15:47:32,760 INFO [StoreOpener-750a447b5bb19a7405a6a02a816d1329-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 750a447b5bb19a7405a6a02a816d1329 columnFamilyName f 2024-12-05T15:47:32,760 DEBUG [StoreOpener-750a447b5bb19a7405a6a02a816d1329-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:32,761 INFO [StoreOpener-750a447b5bb19a7405a6a02a816d1329-1 {}] regionserver.HStore(327): Store=750a447b5bb19a7405a6a02a816d1329/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:47:32,761 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for 750a447b5bb19a7405a6a02a816d1329 2024-12-05T15:47:32,763 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns3/test-17334136463822/750a447b5bb19a7405a6a02a816d1329 2024-12-05T15:47:32,764 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns3/test-17334136463822/750a447b5bb19a7405a6a02a816d1329 2024-12-05T15:47:32,764 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for 750a447b5bb19a7405a6a02a816d1329 2024-12-05T15:47:32,764 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for 750a447b5bb19a7405a6a02a816d1329 2024-12-05T15:47:32,768 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for 750a447b5bb19a7405a6a02a816d1329 2024-12-05T15:47:32,772 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns3/test-17334136463822/750a447b5bb19a7405a6a02a816d1329/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T15:47:32,773 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened 750a447b5bb19a7405a6a02a816d1329; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10601168000, jitterRate=-0.012689292430877686}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T15:47:32,773 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 750a447b5bb19a7405a6a02a816d1329 2024-12-05T15:47:32,774 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for 750a447b5bb19a7405a6a02a816d1329: Running coprocessor pre-open hook at 1733413652755Writing region info on filesystem at 1733413652755Initializing all the Stores at 1733413652757 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413652757Cleaning up temporary data from old regions at 1733413652765 (+8 ms)Running coprocessor post-open hooks at 1733413652773 (+8 ms)Region opened successfully at 1733413652774 (+1 ms) 2024-12-05T15:47:32,775 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329., pid=16, masterSystemTime=1733413652747 2024-12-05T15:47:32,779 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. 2024-12-05T15:47:32,779 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. 2024-12-05T15:47:32,780 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=750a447b5bb19a7405a6a02a816d1329, regionState=OPEN, openSeqNum=2, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:32,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 750a447b5bb19a7405a6a02a816d1329, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:47:32,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-05T15:47:32,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 750a447b5bb19a7405a6a02a816d1329, server=5a29627a94df,34815,1733413637643 in 192 msec 2024-12-05T15:47:32,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-05T15:47:32,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns3:test-17334136463822, region=750a447b5bb19a7405a6a02a816d1329, ASSIGN in 355 msec 2024-12-05T15:47:32,795 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns3:test-17334136463822 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T15:47:32,795 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17334136463822","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413652795"}]},"ts":"1733413652795"} 2024-12-05T15:47:32,798 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17334136463822, state=ENABLED in hbase:meta 2024-12-05T15:47:32,800 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17334136463822 execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T15:47:32,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns3:test-17334136463822 in 1.2060 sec 2024-12-05T15:47:32,913 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17334136463822' 2024-12-05T15:47:32,913 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17334136463821' 2024-12-05T15:47:32,914 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1733413646382' 2024-12-05T15:47:33,600 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-12-05T15:47:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-05T15:47:33,747 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns3:test-17334136463822 completed 2024-12-05T15:47:33,747 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns3:test-17334136463822 get assigned. Timeout = 60000ms 2024-12-05T15:47:33,749 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T15:47:33,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns3:test-17334136463822 assigned to meta. Checking AM states. 2024-12-05T15:47:33,756 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T15:47:33,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns3:test-17334136463822 assigned. 2024-12-05T15:47:33,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns4:test-17334136463823', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:47:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns4:test-17334136463823 2024-12-05T15:47:33,762 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17334136463823 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T15:47:33,763 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:33,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns4" qualifier: "test-17334136463823" procId is: 17 2024-12-05T15:47:33,764 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns4:test-17334136463823 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T15:47:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T15:47:33,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741845_1021 (size=390) 2024-12-05T15:47:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T15:47:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T15:47:34,176 INFO [RegionOpenAndInit-ns4:test-17334136463823-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d2c0907dfdcc66e63e68a2757d1d51ef, NAME => 'ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17334136463823', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:47:34,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741846_1022 (size=50) 2024-12-05T15:47:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T15:47:34,585 DEBUG [RegionOpenAndInit-ns4:test-17334136463823-pool-0 {}] regionserver.HRegion(898): Instantiated ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:34,586 DEBUG [RegionOpenAndInit-ns4:test-17334136463823-pool-0 {}] regionserver.HRegion(1722): Closing d2c0907dfdcc66e63e68a2757d1d51ef, disabling compactions & flushes 2024-12-05T15:47:34,586 INFO [RegionOpenAndInit-ns4:test-17334136463823-pool-0 {}] regionserver.HRegion(1755): Closing region ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. 2024-12-05T15:47:34,586 DEBUG [RegionOpenAndInit-ns4:test-17334136463823-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. 2024-12-05T15:47:34,586 DEBUG [RegionOpenAndInit-ns4:test-17334136463823-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. after waiting 0 ms 2024-12-05T15:47:34,586 DEBUG [RegionOpenAndInit-ns4:test-17334136463823-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. 2024-12-05T15:47:34,586 INFO [RegionOpenAndInit-ns4:test-17334136463823-pool-0 {}] regionserver.HRegion(1973): Closed ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. 2024-12-05T15:47:34,586 DEBUG [RegionOpenAndInit-ns4:test-17334136463823-pool-0 {}] regionserver.HRegion(1676): Region close journal for d2c0907dfdcc66e63e68a2757d1d51ef: Waiting for close lock at 1733413654586Disabling compacts and flushes for region at 1733413654586Disabling writes for close at 1733413654586Writing region close event to WAL at 1733413654586Closed at 1733413654586 2024-12-05T15:47:34,588 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns4:test-17334136463823 execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T15:47:34,588 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1733413654588"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733413654588"}]},"ts":"1733413654588"} 2024-12-05T15:47:34,591 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T15:47:34,593 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns4:test-17334136463823 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T15:47:34,593 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17334136463823","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413654593"}]},"ts":"1733413654593"} 2024-12-05T15:47:34,596 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17334136463823, state=ENABLING in hbase:meta 2024-12-05T15:47:34,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17334136463823, region=d2c0907dfdcc66e63e68a2757d1d51ef, ASSIGN}] 2024-12-05T15:47:34,599 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17334136463823, region=d2c0907dfdcc66e63e68a2757d1d51ef, ASSIGN 2024-12-05T15:47:34,601 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns4:test-17334136463823, region=d2c0907dfdcc66e63e68a2757d1d51ef, ASSIGN; state=OFFLINE, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=false 2024-12-05T15:47:34,752 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=d2c0907dfdcc66e63e68a2757d1d51ef, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:34,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns4:test-17334136463823, region=d2c0907dfdcc66e63e68a2757d1d51ef, ASSIGN because future has completed 2024-12-05T15:47:34,755 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2c0907dfdcc66e63e68a2757d1d51ef, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:47:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T15:47:34,916 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. 2024-12-05T15:47:34,917 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => d2c0907dfdcc66e63e68a2757d1d51ef, NAME => 'ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef.', STARTKEY => '', ENDKEY => ''} 2024-12-05T15:47:34,917 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:47:34,918 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17334136463823 d2c0907dfdcc66e63e68a2757d1d51ef 2024-12-05T15:47:34,918 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:34,918 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for d2c0907dfdcc66e63e68a2757d1d51ef 2024-12-05T15:47:34,918 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for d2c0907dfdcc66e63e68a2757d1d51ef 2024-12-05T15:47:34,920 INFO [StoreOpener-d2c0907dfdcc66e63e68a2757d1d51ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d2c0907dfdcc66e63e68a2757d1d51ef 2024-12-05T15:47:34,923 INFO [StoreOpener-d2c0907dfdcc66e63e68a2757d1d51ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2c0907dfdcc66e63e68a2757d1d51ef columnFamilyName f 2024-12-05T15:47:34,924 DEBUG [StoreOpener-d2c0907dfdcc66e63e68a2757d1d51ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:34,925 INFO [StoreOpener-d2c0907dfdcc66e63e68a2757d1d51ef-1 {}] regionserver.HStore(327): Store=d2c0907dfdcc66e63e68a2757d1d51ef/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:47:34,925 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for d2c0907dfdcc66e63e68a2757d1d51ef 2024-12-05T15:47:34,926 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns4/test-17334136463823/d2c0907dfdcc66e63e68a2757d1d51ef 2024-12-05T15:47:34,927 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns4/test-17334136463823/d2c0907dfdcc66e63e68a2757d1d51ef 2024-12-05T15:47:34,927 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for d2c0907dfdcc66e63e68a2757d1d51ef 2024-12-05T15:47:34,927 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for d2c0907dfdcc66e63e68a2757d1d51ef 2024-12-05T15:47:34,930 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for d2c0907dfdcc66e63e68a2757d1d51ef 2024-12-05T15:47:34,933 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns4/test-17334136463823/d2c0907dfdcc66e63e68a2757d1d51ef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T15:47:34,934 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened d2c0907dfdcc66e63e68a2757d1d51ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11165442880, jitterRate=0.03986290097236633}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T15:47:34,934 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2c0907dfdcc66e63e68a2757d1d51ef 2024-12-05T15:47:34,935 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for d2c0907dfdcc66e63e68a2757d1d51ef: Running coprocessor pre-open hook at 1733413654918Writing region info on filesystem at 1733413654918Initializing all the Stores at 1733413654920 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413654920Cleaning up temporary data from old regions at 1733413654927 (+7 ms)Running coprocessor post-open hooks at 1733413654934 (+7 ms)Region opened successfully at 1733413654935 (+1 ms) 2024-12-05T15:47:34,936 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef., pid=19, masterSystemTime=1733413654908 2024-12-05T15:47:34,939 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. 2024-12-05T15:47:34,940 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. 2024-12-05T15:47:34,941 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=d2c0907dfdcc66e63e68a2757d1d51ef, regionState=OPEN, openSeqNum=2, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:34,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2c0907dfdcc66e63e68a2757d1d51ef, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:47:34,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-05T15:47:34,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure d2c0907dfdcc66e63e68a2757d1d51ef, server=5a29627a94df,34815,1733413637643 in 191 msec 2024-12-05T15:47:34,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-05T15:47:34,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns4:test-17334136463823, region=d2c0907dfdcc66e63e68a2757d1d51ef, ASSIGN in 354 msec 2024-12-05T15:47:34,956 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns4:test-17334136463823 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T15:47:34,957 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17334136463823","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413654956"}]},"ts":"1733413654956"} 2024-12-05T15:47:34,959 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17334136463823, state=ENABLED in hbase:meta 2024-12-05T15:47:34,961 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17334136463823 execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T15:47:34,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns4:test-17334136463823 in 1.2020 sec 2024-12-05T15:47:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T15:47:35,917 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns4:test-17334136463823 completed 2024-12-05T15:47:35,917 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns4:test-17334136463823 get assigned. Timeout = 60000ms 2024-12-05T15:47:35,917 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T15:47:35,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns4:test-17334136463823 assigned to meta. Checking AM states. 2024-12-05T15:47:35,921 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T15:47:35,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns4:test-17334136463823 assigned. 2024-12-05T15:47:35,922 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:47:35,922 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestBackupBase.createTables(TestBackupBase.java:481) at org.apache.hadoop.hbase.backup.TestBackupBase.setUpHelper(TestBackupBase.java:343) at org.apache.hadoop.hbase.backup.TestBackupBase.setUp(TestBackupBase.java:357) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:47:35,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:35,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:35,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:47:35,958 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=610, OpenFileDescriptor=633, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=18, AvailableMemoryMB=9338 2024-12-05T15:47:35,960 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=610 is superior to 500 2024-12-05T15:47:35,961 INFO [Time-limited test {}] backup.TestIncrementalBackup(91): create full backup image for all tables 2024-12-05T15:47:35,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table test-1733413646382 from 'test-1733413646382', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1733413646382', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:47:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=20, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1733413646382 2024-12-05T15:47:36,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741847_1023 (size=908) 2024-12-05T15:47:36,027 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/.tabledesc/.tableinfo.0000000001.367 2024-12-05T15:47:36,027 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/.tabledesc/.tableinfo.0000000002.908 2024-12-05T15:47:36,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1733413646382}] 2024-12-05T15:47:36,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, REOPEN/MOVE}] 2024-12-05T15:47:36,051 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, REOPEN/MOVE 2024-12-05T15:47:36,053 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=24a988b9e3812c25fcebd4f4072c48f0, regionState=CLOSING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:36,055 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33117 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=5a29627a94df,34815,1733413637643, table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T15:47:36,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, REOPEN/MOVE because future has completed 2024-12-05T15:47:36,057 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T15:47:36,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:47:36,217 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(122): Close 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:36,218 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T15:47:36,218 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1722): Closing 24a988b9e3812c25fcebd4f4072c48f0, disabling compactions & flushes 2024-12-05T15:47:36,218 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1755): Closing region test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:36,218 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:36,218 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1843): Acquired close lock on test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. after waiting 0 ms 2024-12-05T15:47:36,218 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1853): Updates disabled for region test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:36,222 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(2902): Flushing 24a988b9e3812c25fcebd4f4072c48f0 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-12-05T15:47:36,288 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/f/4e364da1714b48f9af4efc97be4888c8 is 37, key is row10/f:q1/1733413649096/Put/seqid=0 2024-12-05T15:47:36,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741848_1024 (size=8413) 2024-12-05T15:47:36,572 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1733413646382 2024-12-05T15:47:36,572 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1733413646382 Metrics about Tables on a single HBase RegionServer 2024-12-05T15:47:36,574 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17334136463821 2024-12-05T15:47:36,574 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17334136463821 Metrics about Tables on a single HBase RegionServer 2024-12-05T15:47:36,575 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17334136463823 2024-12-05T15:47:36,575 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17334136463823 Metrics about Tables on a single HBase RegionServer 2024-12-05T15:47:36,576 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17334136463822 2024-12-05T15:47:36,576 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17334136463822 Metrics about Tables on a single HBase RegionServer 2024-12-05T15:47:36,701 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/f/4e364da1714b48f9af4efc97be4888c8 2024-12-05T15:47:36,757 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/f/4e364da1714b48f9af4efc97be4888c8 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8 2024-12-05T15:47:36,766 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8, entries=99, sequenceid=103, filesize=8.2 K 2024-12-05T15:47:36,773 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 24a988b9e3812c25fcebd4f4072c48f0 in 550ms, sequenceid=103, compaction requested=false 2024-12-05T15:47:36,780 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2024-12-05T15:47:36,783 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:47:36,784 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1973): Closed test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:36,784 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1676): Region close journal for 24a988b9e3812c25fcebd4f4072c48f0: Waiting for close lock at 1733413656218Running coprocessor pre-close hooks at 1733413656218Disabling compacts and flushes for region at 1733413656218Disabling writes for close at 1733413656218Obtaining lock to block concurrent updates at 1733413656222 (+4 ms)Preparing flush snapshotting stores in 24a988b9e3812c25fcebd4f4072c48f0 at 1733413656222Finished memstore snapshotting test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0., syncing WAL and waiting on mvcc, flushsize=dataSize=3247, getHeapSize=11248, getOffHeapSize=0, getCellsCount=99 at 1733413656229 (+7 ms)Flushing stores of test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. at 1733413656231 (+2 ms)Flushing 24a988b9e3812c25fcebd4f4072c48f0/f: creating writer at 1733413656233 (+2 ms)Flushing 24a988b9e3812c25fcebd4f4072c48f0/f: appending metadata at 1733413656277 (+44 ms)Flushing 24a988b9e3812c25fcebd4f4072c48f0/f: closing flushed file at 1733413656280 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17e7610d: reopening flushed file at 1733413656755 (+475 ms)Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 24a988b9e3812c25fcebd4f4072c48f0 in 550ms, sequenceid=103, compaction requested=false at 1733413656773 (+18 ms)Writing region close event to WAL at 1733413656775 (+2 ms)Running coprocessor post-close hooks at 1733413656781 (+6 ms)Closed at 1733413656784 (+3 ms) 2024-12-05T15:47:36,784 WARN [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionServer(3299): Not adding moved region record: 24a988b9e3812c25fcebd4f4072c48f0 to self. 2024-12-05T15:47:36,788 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(157): Closed 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:36,788 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=24a988b9e3812c25fcebd4f4072c48f0, regionState=CLOSED 2024-12-05T15:47:36,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:47:36,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-05T15:47:36,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; CloseRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0, server=5a29627a94df,34815,1733413637643 in 736 msec 2024-12-05T15:47:36,797 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, REOPEN/MOVE; state=CLOSED, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=true 2024-12-05T15:47:36,948 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=24a988b9e3812c25fcebd4f4072c48f0, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:36,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, REOPEN/MOVE because future has completed 2024-12-05T15:47:36,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:47:37,119 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:37,119 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 24a988b9e3812c25fcebd4f4072c48f0, NAME => 'test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0.', STARTKEY => '', ENDKEY => ''} 2024-12-05T15:47:37,120 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:47:37,120 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1733413646382 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:37,120 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:37,121 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:37,121 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:37,124 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:37,126 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24a988b9e3812c25fcebd4f4072c48f0 columnFamilyName f 2024-12-05T15:47:37,126 DEBUG [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:37,136 DEBUG [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8 2024-12-05T15:47:37,136 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.HStore(327): Store=24a988b9e3812c25fcebd4f4072c48f0/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:47:37,136 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:37,138 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24a988b9e3812c25fcebd4f4072c48f0 columnFamilyName f3 2024-12-05T15:47:37,138 DEBUG [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:37,139 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.HStore(327): Store=24a988b9e3812c25fcebd4f4072c48f0/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:47:37,140 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:37,147 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24a988b9e3812c25fcebd4f4072c48f0 columnFamilyName mob 2024-12-05T15:47:37,152 DEBUG [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:37,153 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.HStore(327): Store=24a988b9e3812c25fcebd4f4072c48f0/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:47:37,154 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:37,155 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:37,157 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:37,158 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:37,158 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:37,159 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1733413646382 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T15:47:37,161 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:37,162 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 24a988b9e3812c25fcebd4f4072c48f0; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10560190400, jitterRate=-0.01650562882423401}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T15:47:37,162 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:37,163 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 24a988b9e3812c25fcebd4f4072c48f0: Running coprocessor pre-open hook at 1733413657121Writing region info on filesystem at 1733413657121Initializing all the Stores at 1733413657123 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413657123Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413657123Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413657124 (+1 ms)Cleaning up temporary data from old regions at 1733413657158 (+34 ms)Running coprocessor post-open hooks at 1733413657162 (+4 ms)Region opened successfully at 1733413657163 (+1 ms) 2024-12-05T15:47:37,164 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0., pid=24, masterSystemTime=1733413657110 2024-12-05T15:47:37,167 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:37,167 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:37,168 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=24a988b9e3812c25fcebd4f4072c48f0, regionState=OPEN, openSeqNum=107, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:37,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:47:37,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22 2024-12-05T15:47:37,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0, server=5a29627a94df,34815,1733413637643 in 216 msec 2024-12-05T15:47:37,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-05T15:47:37,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, REOPEN/MOVE in 1.1290 sec 2024-12-05T15:47:37,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-05T15:47:37,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1733413646382 in 1.1460 sec 2024-12-05T15:47:37,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1733413646382 in 1.2140 sec 2024-12-05T15:47:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=20 2024-12-05T15:47:37,190 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1733413646382 completed 2024-12-05T15:47:37,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c0a1aea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:37,191 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:47:37,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:47:37,192 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:47:37,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:47:37,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:47:37,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540793fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:37,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:47:37,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:47:37,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:37,194 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:47:37,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4be549d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:37,196 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:47:37,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:47:37,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:47:37,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:47:37,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733413646382', row='row-test-1733413646382-3-0', locateType=CURRENT is [region=test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0., hostname=5a29627a94df,34815,1733413637643, seqNum=107] 2024-12-05T15:47:37,252 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:47:37,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:47:37,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@43c3f558 2024-12-05T15:47:37,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:47:37,257 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:47:37,268 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'backup'} 2024-12-05T15:47:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=backup 2024-12-05T15:47:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T15:47:37,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=backup in 11 msec 2024-12-05T15:47:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T15:47:37,377 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: backup completed 2024-12-05T15:47:37,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:47:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system 2024-12-05T15:47:37,401 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T15:47:37,401 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:37,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2024-12-05T15:47:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-05T15:47:37,402 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T15:47:37,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741849_1025 (size=635) 2024-12-05T15:47:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-05T15:47:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-05T15:47:37,817 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f422dcfec85c6b145613ea93a68881af, NAME => 'backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:47:37,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741850_1026 (size=40) 2024-12-05T15:47:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-05T15:47:38,231 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:38,231 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1722): Closing f422dcfec85c6b145613ea93a68881af, disabling compactions & flushes 2024-12-05T15:47:38,231 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:47:38,232 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:47:38,232 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. after waiting 0 ms 2024-12-05T15:47:38,232 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:47:38,232 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1973): Closed backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:47:38,233 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1676): Region close journal for f422dcfec85c6b145613ea93a68881af: Waiting for close lock at 1733413658231Disabling compacts and flushes for region at 1733413658231Disabling writes for close at 1733413658232 (+1 ms)Writing region close event to WAL at 1733413658232Closed at 1733413658232 2024-12-05T15:47:38,237 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T15:47:38,238 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1733413658237"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733413658237"}]},"ts":"1733413658237"} 2024-12-05T15:47:38,241 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T15:47:38,242 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T15:47:38,243 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413658242"}]},"ts":"1733413658242"} 2024-12-05T15:47:38,246 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-12-05T15:47:38,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=f422dcfec85c6b145613ea93a68881af, ASSIGN}] 2024-12-05T15:47:38,248 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=f422dcfec85c6b145613ea93a68881af, ASSIGN 2024-12-05T15:47:38,249 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=f422dcfec85c6b145613ea93a68881af, ASSIGN; state=OFFLINE, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=false 2024-12-05T15:47:38,401 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=f422dcfec85c6b145613ea93a68881af, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:38,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=f422dcfec85c6b145613ea93a68881af, ASSIGN because future has completed 2024-12-05T15:47:38,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure f422dcfec85c6b145613ea93a68881af, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:47:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-05T15:47:38,568 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:47:38,568 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => f422dcfec85c6b145613ea93a68881af, NAME => 'backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af.', STARTKEY => '', ENDKEY => ''} 2024-12-05T15:47:38,568 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:47:38,568 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:38,569 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:38,569 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:38,569 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:38,572 INFO [StoreOpener-f422dcfec85c6b145613ea93a68881af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:38,575 INFO [StoreOpener-f422dcfec85c6b145613ea93a68881af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f422dcfec85c6b145613ea93a68881af columnFamilyName meta 2024-12-05T15:47:38,575 DEBUG [StoreOpener-f422dcfec85c6b145613ea93a68881af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:38,576 INFO [StoreOpener-f422dcfec85c6b145613ea93a68881af-1 {}] regionserver.HStore(327): Store=f422dcfec85c6b145613ea93a68881af/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:47:38,576 INFO [StoreOpener-f422dcfec85c6b145613ea93a68881af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:38,579 INFO [StoreOpener-f422dcfec85c6b145613ea93a68881af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f422dcfec85c6b145613ea93a68881af columnFamilyName session 2024-12-05T15:47:38,579 DEBUG [StoreOpener-f422dcfec85c6b145613ea93a68881af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:38,580 INFO [StoreOpener-f422dcfec85c6b145613ea93a68881af-1 {}] regionserver.HStore(327): Store=f422dcfec85c6b145613ea93a68881af/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:47:38,580 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:38,581 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:38,582 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:38,583 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:38,583 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:38,583 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-05T15:47:38,585 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:38,588 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T15:47:38,588 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened f422dcfec85c6b145613ea93a68881af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10718160320, jitterRate=-0.0017935335636138916}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-05T15:47:38,589 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:38,589 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for f422dcfec85c6b145613ea93a68881af: Running coprocessor pre-open hook at 1733413658569Writing region info on filesystem at 1733413658569Initializing all the Stores at 1733413658571 (+2 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413658571Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413658571Cleaning up temporary data from old regions at 1733413658583 (+12 ms)Running coprocessor post-open hooks at 1733413658589 (+6 ms)Region opened successfully at 1733413658589 2024-12-05T15:47:38,590 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., pid=28, masterSystemTime=1733413658561 2024-12-05T15:47:38,593 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:47:38,593 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:47:38,594 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=f422dcfec85c6b145613ea93a68881af, regionState=OPEN, openSeqNum=2, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:38,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure f422dcfec85c6b145613ea93a68881af, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:47:38,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-05T15:47:38,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure f422dcfec85c6b145613ea93a68881af, server=5a29627a94df,34815,1733413637643 in 191 msec 2024-12-05T15:47:38,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-05T15:47:38,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=f422dcfec85c6b145613ea93a68881af, ASSIGN in 354 msec 2024-12-05T15:47:38,605 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T15:47:38,605 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413658605"}]},"ts":"1733413658605"} 2024-12-05T15:47:38,607 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-12-05T15:47:38,609 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T15:47:38,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system in 1.2100 sec 2024-12-05T15:47:38,916 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17334136463823' 2024-12-05T15:47:38,918 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-12-05T15:47:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-05T15:47:39,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system completed 2024-12-05T15:47:39,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:47:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system_bulk 2024-12-05T15:47:39,573 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T15:47:39,573 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:39,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2024-12-05T15:47:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T15:47:39,574 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T15:47:39,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741851_1027 (size=640) 2024-12-05T15:47:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T15:47:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T15:47:39,984 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eb162556a9ba10393a1061e0fd710298, NAME => 'backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:47:39,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741852_1028 (size=45) 2024-12-05T15:47:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T15:47:40,394 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:40,394 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1722): Closing eb162556a9ba10393a1061e0fd710298, disabling compactions & flushes 2024-12-05T15:47:40,394 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. 2024-12-05T15:47:40,394 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. 2024-12-05T15:47:40,394 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. after waiting 0 ms 2024-12-05T15:47:40,394 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. 2024-12-05T15:47:40,394 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1973): Closed backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. 2024-12-05T15:47:40,395 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1676): Region close journal for eb162556a9ba10393a1061e0fd710298: Waiting for close lock at 1733413660394Disabling compacts and flushes for region at 1733413660394Disabling writes for close at 1733413660394Writing region close event to WAL at 1733413660394Closed at 1733413660394 2024-12-05T15:47:40,397 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T15:47:40,397 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733413660397"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733413660397"}]},"ts":"1733413660397"} 2024-12-05T15:47:40,402 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T15:47:40,403 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T15:47:40,404 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413660404"}]},"ts":"1733413660404"} 2024-12-05T15:47:40,406 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2024-12-05T15:47:40,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=eb162556a9ba10393a1061e0fd710298, ASSIGN}] 2024-12-05T15:47:40,407 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=eb162556a9ba10393a1061e0fd710298, ASSIGN 2024-12-05T15:47:40,408 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=eb162556a9ba10393a1061e0fd710298, ASSIGN; state=OFFLINE, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=false 2024-12-05T15:47:40,421 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-12-05T15:47:40,559 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=eb162556a9ba10393a1061e0fd710298, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:40,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=eb162556a9ba10393a1061e0fd710298, ASSIGN because future has completed 2024-12-05T15:47:40,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb162556a9ba10393a1061e0fd710298, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:47:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T15:47:40,720 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. 2024-12-05T15:47:40,720 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => eb162556a9ba10393a1061e0fd710298, NAME => 'backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298.', STARTKEY => '', ENDKEY => ''} 2024-12-05T15:47:40,721 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:47:40,721 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk eb162556a9ba10393a1061e0fd710298 2024-12-05T15:47:40,721 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:47:40,721 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for eb162556a9ba10393a1061e0fd710298 2024-12-05T15:47:40,721 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for eb162556a9ba10393a1061e0fd710298 2024-12-05T15:47:40,724 INFO [StoreOpener-eb162556a9ba10393a1061e0fd710298-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region eb162556a9ba10393a1061e0fd710298 2024-12-05T15:47:40,725 INFO [StoreOpener-eb162556a9ba10393a1061e0fd710298-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb162556a9ba10393a1061e0fd710298 columnFamilyName meta 2024-12-05T15:47:40,725 DEBUG [StoreOpener-eb162556a9ba10393a1061e0fd710298-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:40,726 INFO [StoreOpener-eb162556a9ba10393a1061e0fd710298-1 {}] regionserver.HStore(327): Store=eb162556a9ba10393a1061e0fd710298/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:47:40,726 INFO [StoreOpener-eb162556a9ba10393a1061e0fd710298-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region eb162556a9ba10393a1061e0fd710298 2024-12-05T15:47:40,728 INFO [StoreOpener-eb162556a9ba10393a1061e0fd710298-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb162556a9ba10393a1061e0fd710298 columnFamilyName session 2024-12-05T15:47:40,728 DEBUG [StoreOpener-eb162556a9ba10393a1061e0fd710298-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:40,729 INFO [StoreOpener-eb162556a9ba10393a1061e0fd710298-1 {}] regionserver.HStore(327): Store=eb162556a9ba10393a1061e0fd710298/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:47:40,729 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for eb162556a9ba10393a1061e0fd710298 2024-12-05T15:47:40,731 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system_bulk/eb162556a9ba10393a1061e0fd710298 2024-12-05T15:47:40,731 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system_bulk/eb162556a9ba10393a1061e0fd710298 2024-12-05T15:47:40,732 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for eb162556a9ba10393a1061e0fd710298 2024-12-05T15:47:40,733 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for eb162556a9ba10393a1061e0fd710298 2024-12-05T15:47:40,733 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-05T15:47:40,735 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for eb162556a9ba10393a1061e0fd710298 2024-12-05T15:47:40,739 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system_bulk/eb162556a9ba10393a1061e0fd710298/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T15:47:40,740 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened eb162556a9ba10393a1061e0fd710298; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12057745760, jitterRate=0.12296508252620697}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-05T15:47:40,740 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb162556a9ba10393a1061e0fd710298 2024-12-05T15:47:40,740 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for eb162556a9ba10393a1061e0fd710298: Running coprocessor pre-open hook at 1733413660721Writing region info on filesystem at 1733413660722 (+1 ms)Initializing all the Stores at 1733413660723 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413660723Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413660723Cleaning up temporary data from old regions at 1733413660733 (+10 ms)Running coprocessor post-open hooks at 1733413660740 (+7 ms)Region opened successfully at 1733413660740 2024-12-05T15:47:40,741 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298., pid=31, masterSystemTime=1733413660715 2024-12-05T15:47:40,744 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. 2024-12-05T15:47:40,744 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. 2024-12-05T15:47:40,745 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=eb162556a9ba10393a1061e0fd710298, regionState=OPEN, openSeqNum=2, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:47:40,747 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb162556a9ba10393a1061e0fd710298, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:47:40,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-05T15:47:40,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure eb162556a9ba10393a1061e0fd710298, server=5a29627a94df,34815,1733413637643 in 187 msec 2024-12-05T15:47:40,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-05T15:47:40,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=eb162556a9ba10393a1061e0fd710298, ASSIGN in 346 msec 2024-12-05T15:47:40,756 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T15:47:40,756 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413660756"}]},"ts":"1733413660756"} 2024-12-05T15:47:40,759 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2024-12-05T15:47:40,760 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T15:47:40,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system_bulk in 1.1900 sec 2024-12-05T15:47:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T15:47:41,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system_bulk completed 2024-12-05T15:47:41,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:47:41,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:47:41,759 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-05T15:47:41,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='activesession:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:47:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2024-12-05T15:47:41,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-05T15:47:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733413661793 (current time:1733413661793). 2024-12-05T15:47:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-05T15:47:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-05T15:47:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71ce3b39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:47:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:47:41,795 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:47:41,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:47:41,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:47:41,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688ce502, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:41,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:47:41,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:47:41,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:41,797 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:47:41,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf64b90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:47:41,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:47:41,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:47:41,802 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:47:41,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117. 2024-12-05T15:47:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:47:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:41,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:47:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-05T15:47:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-05T15:47:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-12-05T15:47:41,812 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-05T15:47:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-05T15:47:41,817 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-05T15:47:41,828 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-05T15:47:41,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741853_1029 (size=52) 2024-12-05T15:47:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-05T15:47:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-05T15:47:42,238 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-05T15:47:42,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af}] 2024-12-05T15:47:42,249 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:42,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34815 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-12-05T15:47:42,412 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:47:42,413 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing f422dcfec85c6b145613ea93a68881af 2/2 column families, dataSize=45 B heapSize=632 B 2024-12-05T15:47:42,431 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/session/ff33f5218d644a85986a494156767a9c is 49, key is activesession:/session:c/1733413661781/Put/seqid=0 2024-12-05T15:47:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-05T15:47:42,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741854_1030 (size=5049) 2024-12-05T15:47:42,839 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/session/ff33f5218d644a85986a494156767a9c 2024-12-05T15:47:42,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/session/ff33f5218d644a85986a494156767a9c as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/ff33f5218d644a85986a494156767a9c 2024-12-05T15:47:42,863 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/ff33f5218d644a85986a494156767a9c, entries=1, sequenceid=5, filesize=4.9 K 2024-12-05T15:47:42,865 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for f422dcfec85c6b145613ea93a68881af in 451ms, sequenceid=5, compaction requested=false 2024-12-05T15:47:42,865 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for f422dcfec85c6b145613ea93a68881af: 2024-12-05T15:47:42,865 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. for snapshot_backup_system completed. 2024-12-05T15:47:42,866 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af.' region-info for snapshot=snapshot_backup_system 2024-12-05T15:47:42,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-05T15:47:42,871 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-05T15:47:42,871 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/ff33f5218d644a85986a494156767a9c] hfiles 2024-12-05T15:47:42,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/ff33f5218d644a85986a494156767a9c for snapshot=snapshot_backup_system 2024-12-05T15:47:42,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741855_1031 (size=95) 2024-12-05T15:47:42,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-05T15:47:43,284 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:47:43,289 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-12-05T15:47:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-12-05T15:47:43,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:43,292 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:43,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-05T15:47:43,296 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-05T15:47:43,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af in 1.0500 sec 2024-12-05T15:47:43,298 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-05T15:47:43,299 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-05T15:47:43,299 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-05T15:47:43,301 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-05T15:47:43,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741856_1032 (size=731) 2024-12-05T15:47:43,726 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-05T15:47:43,739 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-05T15:47:43,740 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_backup_system 2024-12-05T15:47:43,742 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-05T15:47:43,742 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-12-05T15:47:43,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.9340 sec 2024-12-05T15:47:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-05T15:47:43,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-05T15:47:43,960 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733413657238 started at 1733413663958. 2024-12-05T15:47:43,978 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733413657238 has been started. 2024-12-05T15:47:43,982 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-12-05T15:47:43,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-12-05T15:47:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-05T15:47:43,990 INFO [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-05T15:47:43,990 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-05T15:47:43,991 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-05T15:47:43,991 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-05T15:47:43,992 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-05T15:47:43,992 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-05T15:47:44,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-05T15:47:44,036 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,036 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-05T15:47:44,036 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-05T15:47:44,037 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,037 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-05T15:47:44,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-05T15:47:44,039 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-05T15:47:44,041 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-05T15:47:44,041 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-05T15:47:44,041 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-05T15:47:44,044 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-05T15:47:44,044 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-05T15:47:44,045 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-05T15:47:44,045 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-05T15:47:44,045 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-05T15:47:44,045 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-05T15:47:44,045 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '5a29627a94df,34815,1733413637643' joining acquired barrier for procedure (rolllog) in zk 2024-12-05T15:47:44,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,052 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-05T15:47:44,052 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,052 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-05T15:47:44,052 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-05T15:47:44,052 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-05T15:47:44,052 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-05T15:47:44,053 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-05T15:47:44,053 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-05T15:47:44,053 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:47:44,054 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,054 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-05T15:47:44,055 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '5a29627a94df,34815,1733413637643' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-05T15:47:44,055 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-05T15:47:44,055 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@62092801[Count = 0] remaining members to acquire global barrier 2024-12-05T15:47:44,055 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-05T15:47:44,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-05T15:47:44,060 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-05T15:47:44,060 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-05T15:47:44,060 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-05T15:47:44,060 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,061 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-05T15:47:44,061 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-05T15:47:44,062 DEBUG [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,068 INFO [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733413641091 highest: 1733413648077 on 5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,068 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1:(num 1733413648077) roll requested 2024-12-05T15:47:44,080 DEBUG [regionserver/5a29627a94df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413664068, exclude list is [], retry=0 2024-12-05T15:47:44,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:47:44,086 INFO [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413648077 with entries=24, filesize=4.64 KB; new WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413664068 2024-12-05T15:47:44,087 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:47:44,087 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413648077 is not closed yet, will try archiving it next time 2024-12-05T15:47:44,087 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0:(num 1733413641091) roll requested 2024-12-05T15:47:44,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741840_1016 (size=4756) 2024-12-05T15:47:44,100 DEBUG [regionserver/5a29627a94df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413664087, exclude list is [], retry=0 2024-12-05T15:47:44,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:47:44,106 INFO [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413641091 with entries=3, filesize=1.00 KB; new WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413664087 2024-12-05T15:47:44,106 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:47:44,106 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413641091 is not closed yet, will try archiving it next time 2024-12-05T15:47:44,106 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5a29627a94df%2C34815%2C1733413637643.meta:.meta(num 1733413641984) roll requested 2024-12-05T15:47:44,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741833_1009 (size=1034) 2024-12-05T15:47:44,108 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413641091 not finished, retry = 0 2024-12-05T15:47:44,119 DEBUG [regionserver/5a29627a94df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413664106.meta, exclude list is [], retry=0 2024-12-05T15:47:44,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:47:44,124 INFO [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413641984.meta with entries=42, filesize=11.76 KB; new WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413664106.meta 2024-12-05T15:47:44,125 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:47:44,125 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413641984.meta is not closed yet, will try archiving it next time 2024-12-05T15:47:44,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741834_1010 (size=12055) 2024-12-05T15:47:44,127 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413641984.meta not finished, retry = 0 2024-12-05T15:47:44,128 DEBUG [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-12-05T15:47:44,129 INFO [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733413664087 on 5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,129 DEBUG [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:47:44,129 DEBUG [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:47:44,129 DEBUG [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@42c33eba 2024-12-05T15:47:44,129 DEBUG [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:47:44,130 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45939, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-05T15:47:44,131 DEBUG [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:47:44,131 DEBUG [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:47:44,132 DEBUG [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:47:44,133 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40419, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-05T15:47:44,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:47:44,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:47:44,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='rslogts:hdfs://localhost:37233/backupUT\x00', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:47:44,159 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-05T15:47:44,159 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '5a29627a94df,34815,1733413637643' in zk 2024-12-05T15:47:44,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,169 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-05T15:47:44,169 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-05T15:47:44,169 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,169 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-05T15:47:44,169 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-05T15:47:44,169 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-05T15:47:44,170 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-05T15:47:44,170 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-05T15:47:44,171 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:47:44,171 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-05T15:47:44,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:47:44,173 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,174 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '5a29627a94df,34815,1733413637643': 2024-12-05T15:47:44,174 INFO [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-05T15:47:44,174 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-05T15:47:44,174 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-05T15:47:44,174 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '5a29627a94df,34815,1733413637643' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-05T15:47:44,174 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-05T15:47:44,174 INFO [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-05T15:47:44,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-05T15:47:44,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-05T15:47:44,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-05T15:47:44,185 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-05T15:47:44,185 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-05T15:47:44,185 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-05T15:47:44,185 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-05T15:47:44,185 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-05T15:47:44,186 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-05T15:47:44,186 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-05T15:47:44,186 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,186 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-05T15:47:44,186 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:47:44,186 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-05T15:47:44,187 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-05T15:47:44,187 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:47:44,187 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,188 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,188 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-05T15:47:44,188 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:47:44,189 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-05T15:47:44,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-05T15:47:44,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-05T15:47:44,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:47:44,202 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-05T15:47:44,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-05T15:47:44,202 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-05T15:47:44,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-05T15:47:44,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-05T15:47:44,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-05T15:47:44,203 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-05T15:47:44,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-05T15:47:44,203 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-05T15:47:44,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-05T15:47:44,203 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-05T15:47:44,210 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413641091 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413641091 2024-12-05T15:47:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-05T15:47:44,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733413664223_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } 2024-12-05T15:47:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733413664224 (current time:1733413664224). 2024-12-05T15:47:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-05T15:47:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733413664223_default_test-1733413646382 VERSION not specified, setting to 2 2024-12-05T15:47:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d85ef36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:47:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:47:44,226 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:47:44,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:47:44,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:47:44,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff88dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:44,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:47:44,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:47:44,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:44,228 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:47:44,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d84ab5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:47:44,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:47:44,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:47:44,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:47:44,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117. 2024-12-05T15:47:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:47:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:44,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:47:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-05T15:47:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733413664223_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } 2024-12-05T15:47:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733413664223_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-12-05T15:47:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T15:47:44,237 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733413664223_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-05T15:47:44,238 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733413664223_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-05T15:47:44,242 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733413664223_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-05T15:47:44,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741860_1036 (size=84) 2024-12-05T15:47:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T15:47:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T15:47:44,654 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733413664223_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-05T15:47:44,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0}] 2024-12-05T15:47:44,657 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:44,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-12-05T15:47:44,811 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:44,812 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 24a988b9e3812c25fcebd4f4072c48f0 3/3 column families, dataSize=654 B heapSize=2.25 KB 2024-12-05T15:47:44,833 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/f3/c7bc8ee8395f42e39c50f2005338ee85 is 58, key is row-test-1733413646382-3-0/f3:q1/1733413657204/Put/seqid=0 2024-12-05T15:47:44,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741861_1037 (size=5387) 2024-12-05T15:47:44,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T15:47:44,920 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2024-12-05T15:47:45,240 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/f3/c7bc8ee8395f42e39c50f2005338ee85 2024-12-05T15:47:45,266 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T15:47:45,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741862_1038 (size=4469) 2024-12-05T15:47:45,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T15:47:45,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/mob/09842ffb0751416eaaeffe3bff7aefa1 is 59, key is row-test-1733413646382-3-0/mob:q1/1733413657221/Put/seqid=0 2024-12-05T15:47:45,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741863_1039 (size=5484) 2024-12-05T15:47:46,109 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=330, hasBloomFilter=true, into tmp file hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/mob/09842ffb0751416eaaeffe3bff7aefa1 2024-12-05T15:47:46,120 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/f3/c7bc8ee8395f42e39c50f2005338ee85 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f3/c7bc8ee8395f42e39c50f2005338ee85 2024-12-05T15:47:46,130 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f3/c7bc8ee8395f42e39c50f2005338ee85, entries=6, sequenceid=121, filesize=5.3 K 2024-12-05T15:47:46,132 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/mob/09842ffb0751416eaaeffe3bff7aefa1 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/09842ffb0751416eaaeffe3bff7aefa1 2024-12-05T15:47:46,143 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/09842ffb0751416eaaeffe3bff7aefa1, entries=6, sequenceid=121, filesize=5.4 K 2024-12-05T15:47:46,144 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~654 B/654, heapSize ~1.97 KB/2016, currentSize=0 B/0 for 24a988b9e3812c25fcebd4f4072c48f0 in 1332ms, sequenceid=121, compaction requested=false 2024-12-05T15:47:46,144 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 24a988b9e3812c25fcebd4f4072c48f0: 2024-12-05T15:47:46,144 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. for snapshot_1733413664223_default_test-1733413646382 completed. 2024-12-05T15:47:46,145 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0.' region-info for snapshot=snapshot_1733413664223_default_test-1733413646382 2024-12-05T15:47:46,145 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-05T15:47:46,145 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8] hfiles 2024-12-05T15:47:46,145 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8 for snapshot=snapshot_1733413664223_default_test-1733413646382 2024-12-05T15:47:46,145 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f3/c7bc8ee8395f42e39c50f2005338ee85] hfiles 2024-12-05T15:47:46,145 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f3/c7bc8ee8395f42e39c50f2005338ee85 for snapshot=snapshot_1733413664223_default_test-1733413646382 2024-12-05T15:47:46,145 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/09842ffb0751416eaaeffe3bff7aefa1] hfiles 2024-12-05T15:47:46,145 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/09842ffb0751416eaaeffe3bff7aefa1 for snapshot=snapshot_1733413664223_default_test-1733413646382 2024-12-05T15:47:46,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741864_1040 (size=185) 2024-12-05T15:47:46,223 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-12-05T15:47:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T15:47:46,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:46,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-12-05T15:47:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-12-05T15:47:46,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733413664223_default_test-1733413646382 on region 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:46,558 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:46,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-05T15:47:46,567 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733413664223_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-05T15:47:46,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0 in 1.9080 sec 2024-12-05T15:47:46,568 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733413664223_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-05T15:47:46,571 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-12-05T15:47:46,571 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-12-05T15:47:46,572 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:46,572 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-12-05T15:47:46,572 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2024-12-05T15:47:46,573 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-12-05T15:47:46,573 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-12-05T15:47:46,573 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2024-12-05T15:47:46,573 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:46,574 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f3 2024-12-05T15:47:46,574 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:46,575 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: mob 2024-12-05T15:47:46,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741865_1041 (size=49) 2024-12-05T15:47:47,019 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733413664223_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-05T15:47:47,019 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733413664223_default_test-1733413646382 2024-12-05T15:47:47,021 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_1733413664223_default_test-1733413646382 2024-12-05T15:47:47,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741866_1042 (size=1146) 2024-12-05T15:47:47,446 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733413664223_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-05T15:47:47,458 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733413664223_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-05T15:47:47,458 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_1733413664223_default_test-1733413646382 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_1733413664223_default_test-1733413646382 2024-12-05T15:47:47,460 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733413664223_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-05T15:47:47,460 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733413664223_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-12-05T15:47:47,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733413664223_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } in 3.2260 sec 2024-12-05T15:47:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T15:47:48,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1733413646382 completed 2024-12-05T15:47:48,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733413668406_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } 2024-12-05T15:47:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733413668407 (current time:1733413668407). 2024-12-05T15:47:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-05T15:47:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733413668406_ns2_test-17334136463821 VERSION not specified, setting to 2 2024-12-05T15:47:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ff0c088, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:47:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:47:48,409 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:47:48,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:47:48,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:47:48,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26802f6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:48,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:47:48,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:47:48,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:48,410 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:47:48,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf5f318, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:47:48,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:47:48,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:47:48,414 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:47:48,415 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117. 2024-12-05T15:47:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:47:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:48,416 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:47:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-05T15:47:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733413668406_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } 2024-12-05T15:47:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733413668406_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-12-05T15:47:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T15:47:48,419 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733413668406_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-05T15:47:48,421 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733413668406_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-05T15:47:48,425 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733413668406_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-05T15:47:48,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741867_1043 (size=86) 2024-12-05T15:47:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T15:47:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T15:47:48,832 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733413668406_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-05T15:47:48,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure b7877185d3b50523892995990a3cd728}] 2024-12-05T15:47:48,834 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure b7877185d3b50523892995990a3cd728 2024-12-05T15:47:48,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2024-12-05T15:47:48,987 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:47:48,987 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing b7877185d3b50523892995990a3cd728 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-12-05T15:47:49,012 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/.tmp/f/1159a028de3844c4b30999226eb32483 is 37, key is row10/f:q1/1733413651450/Put/seqid=0 2024-12-05T15:47:49,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741868_1044 (size=8413) 2024-12-05T15:47:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T15:47:49,418 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/.tmp/f/1159a028de3844c4b30999226eb32483 2024-12-05T15:47:49,429 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/.tmp/f/1159a028de3844c4b30999226eb32483 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/1159a028de3844c4b30999226eb32483 2024-12-05T15:47:49,436 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/1159a028de3844c4b30999226eb32483, entries=99, sequenceid=103, filesize=8.2 K 2024-12-05T15:47:49,437 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for b7877185d3b50523892995990a3cd728 in 450ms, sequenceid=103, compaction requested=false 2024-12-05T15:47:49,437 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for b7877185d3b50523892995990a3cd728: 2024-12-05T15:47:49,437 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. for snapshot_1733413668406_ns2_test-17334136463821 completed. 2024-12-05T15:47:49,437 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728.' region-info for snapshot=snapshot_1733413668406_ns2_test-17334136463821 2024-12-05T15:47:49,437 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-05T15:47:49,437 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/1159a028de3844c4b30999226eb32483] hfiles 2024-12-05T15:47:49,437 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/1159a028de3844c4b30999226eb32483 for snapshot=snapshot_1733413668406_ns2_test-17334136463821 2024-12-05T15:47:49,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741869_1045 (size=91) 2024-12-05T15:47:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T15:47:49,844 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:47:49,844 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-05T15:47:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-05T15:47:49,845 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733413668406_ns2_test-17334136463821 on region b7877185d3b50523892995990a3cd728 2024-12-05T15:47:49,845 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure b7877185d3b50523892995990a3cd728 2024-12-05T15:47:49,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-05T15:47:49,848 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733413668406_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-05T15:47:49,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; SnapshotRegionProcedure b7877185d3b50523892995990a3cd728 in 1.0130 sec 2024-12-05T15:47:49,849 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733413668406_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-05T15:47:49,850 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733413668406_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-05T15:47:49,850 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733413668406_ns2_test-17334136463821 2024-12-05T15:47:49,851 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_1733413668406_ns2_test-17334136463821 2024-12-05T15:47:49,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741870_1046 (size=456) 2024-12-05T15:47:50,263 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733413668406_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-05T15:47:50,270 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733413668406_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-05T15:47:50,271 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_1733413668406_ns2_test-17334136463821 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_1733413668406_ns2_test-17334136463821 2024-12-05T15:47:50,273 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733413668406_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-05T15:47:50,273 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733413668406_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-12-05T15:47:50,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733413668406_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } in 1.8560 sec 2024-12-05T15:47:50,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T15:47:50,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17334136463821 completed 2024-12-05T15:47:50,568 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1733413657238 2024-12-05T15:47:50,568 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-12-05T15:47:50,571 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2024-12-05T15:47:50,575 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733413657238_ns2:test-17334136463821 2024-12-05T15:47:50,576 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733413668406_ns2_test-17334136463821 to hdfs://localhost:37233/backupUT/backup_1733413657238/ns2/test-17334136463821/ 2024-12-05T15:47:50,584 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-05T15:47:50,619 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:37233, inputRoot=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:47:50,619 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_51721194_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37233/backupUT/backup_1733413657238/ns2/test-17334136463821, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37233/backupUT/backup_1733413657238/ns2/test-17334136463821/.hbase-snapshot/.tmp/snapshot_1733413668406_ns2_test-17334136463821 2024-12-05T15:47:50,622 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-05T15:47:50,628 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_1733413668406_ns2_test-17334136463821 to hdfs://localhost:37233/backupUT/backup_1733413657238/ns2/test-17334136463821/.hbase-snapshot/.tmp/snapshot_1733413668406_ns2_test-17334136463821 2024-12-05T15:47:50,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741871_1047 (size=86) 2024-12-05T15:47:50,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741872_1048 (size=456) 2024-12-05T15:47:51,047 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:51,048 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:51,048 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:51,841 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-16261674832767488650.jar 2024-12-05T15:47:51,841 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:51,842 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:51,842 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:51,842 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:51,843 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:51,843 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:51,843 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:51,844 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:51,844 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:51,844 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:47:51,845 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-05T15:47:51,845 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-05T15:47:51,845 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-05T15:47:51,846 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-05T15:47:51,846 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-05T15:47:51,846 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-05T15:47:51,846 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-05T15:47:51,847 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-05T15:47:51,847 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-05T15:47:51,847 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-05T15:47:51,849 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:47:51,849 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:47:51,849 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:47:51,849 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:47:51,850 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:47:51,850 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:47:51,850 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:47:51,906 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:47:51,936 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733413668406_ns2_test-17334136463821' hfile list 2024-12-05T15:47:51,942 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=8.2 K 2024-12-05T15:47:52,144 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local1323672473_0001_abe30fe0-fd16-4319-93ce-065f94639697/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-05T15:47:52,144 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local1323672473_0001_abe30fe0-fd16-4319-93ce-065f94639697/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-05T15:47:52,264 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-05T15:47:52,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741873_1049 (size=8413) 2024-12-05T15:47:52,743 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/1159a028de3844c4b30999226eb32483 output=hdfs://localhost:37233/backupUT/backup_1733413657238/ns2/test-17334136463821/archive/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/1159a028de3844c4b30999226eb32483 2024-12-05T15:47:52,745 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.017M/sec 2024-12-05T15:47:53,208 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-05T15:47:53,209 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-05T15:47:53,216 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733413668406_ns2_test-17334136463821 2024-12-05T15:47:53,216 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733413668406_ns2_test-17334136463821 finished. 2024-12-05T15:47:53,216 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733413657238_test-1733413646382 2024-12-05T15:47:53,216 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733413664223_default_test-1733413646382 to hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/ 2024-12-05T15:47:53,216 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-05T15:47:53,243 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:37233, inputRoot=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:47:53,243 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_51721194_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/.hbase-snapshot/.tmp/snapshot_1733413664223_default_test-1733413646382 2024-12-05T15:47:53,246 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-05T15:47:53,253 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_1733413664223_default_test-1733413646382 to hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/.hbase-snapshot/.tmp/snapshot_1733413664223_default_test-1733413646382 2024-12-05T15:47:53,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741874_1050 (size=84) 2024-12-05T15:47:53,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741875_1051 (size=1146) 2024-12-05T15:47:53,669 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:53,669 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:53,670 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:54,452 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-3686437698006549541.jar 2024-12-05T15:47:54,452 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:54,452 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:54,453 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:54,453 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:54,453 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:54,453 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:54,453 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:54,454 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:54,454 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:47:54,454 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:47:54,454 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-05T15:47:54,454 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-05T15:47:54,455 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-05T15:47:54,455 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-05T15:47:54,455 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-05T15:47:54,455 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-05T15:47:54,456 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-05T15:47:54,456 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-05T15:47:54,456 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-05T15:47:54,456 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-05T15:47:54,457 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:47:54,457 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:47:54,457 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:47:54,457 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:47:54,457 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:47:54,458 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:47:54,458 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:47:54,492 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:47:54,517 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733413664223_default_test-1733413646382' hfile list 2024-12-05T15:47:54,520 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=18.8 K 2024-12-05T15:47:54,600 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local1732684482_0002_58b1633e-d46c-40f9-a016-8c99cc5acb56/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-05T15:47:54,600 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local1732684482_0002_58b1633e-d46c-40f9-a016-8c99cc5acb56/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-05T15:47:54,687 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-05T15:47:54,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741876_1052 (size=8413) 2024-12-05T15:47:55,128 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8 output=hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8 2024-12-05T15:47:55,129 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-12-05T15:47:55,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741877_1053 (size=5484) 2024-12-05T15:47:55,588 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/09842ffb0751416eaaeffe3bff7aefa1 output=hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/09842ffb0751416eaaeffe3bff7aefa1 2024-12-05T15:47:55,588 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-12-05T15:47:55,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741878_1054 (size=5387) 2024-12-05T15:47:56,036 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f3/c7bc8ee8395f42e39c50f2005338ee85 output=hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f3/c7bc8ee8395f42e39c50f2005338ee85 2024-12-05T15:47:56,036 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5387 (5.3 K) time=0sec 0.012M/sec 2024-12-05T15:47:56,660 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-05T15:47:56,661 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-05T15:47:56,672 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733413664223_default_test-1733413646382 2024-12-05T15:47:56,672 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733413664223_default_test-1733413646382 finished. 2024-12-05T15:47:56,672 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17334136463821 2024-12-05T15:47:56,673 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1733413646382 2024-12-05T15:47:56,718 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733413657238 2024-12-05T15:47:56,718 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-12-05T15:47:56,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741879_1055 (size=97) 2024-12-05T15:47:57,129 INFO [Time-limited test {}] impl.BackupManifest(504): Manifest file stored to hdfs://localhost:37233/backupUT/backup_1733413657238/.backup.manifest 2024-12-05T15:47:57,133 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1733413657238 finished: type=FULL,tablelist=ns2:test-17334136463821;test-1733413646382,targetRootDir=hdfs://localhost:37233/backupUT,startts=1733413663958,completets=1733413676717,bytescopied=0 2024-12-05T15:47:57,133 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-12-05T15:47:57,133 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733413668406_ns2_test-17334136463821 2024-12-05T15:47:57,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733413668406_ns2_test-17334136463821" type: DISABLED 2024-12-05T15:47:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733413668406_ns2_test-17334136463821 2024-12-05T15:47:57,147 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733413668406_ns2_test-17334136463821 for backup backup_1733413657238 succeeded. 2024-12-05T15:47:57,147 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733413664223_default_test-1733413646382 2024-12-05T15:47:57,148 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733413664223_default_test-1733413646382" type: DISABLED 2024-12-05T15:47:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733413664223_default_test-1733413646382 2024-12-05T15:47:57,152 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733413664223_default_test-1733413646382 for backup backup_1733413657238 succeeded. 2024-12-05T15:47:57,153 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-05T15:47:57,164 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-12-05T15:47:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-05T15:47:57,168 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-05T15:47:57,173 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-05T15:47:57,189 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1733413657238 completed. 2024-12-05T15:47:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:47:57,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:47:57,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:backup_1733413657238', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:47:57,213 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:37233/backupUT/backup_1733413657238/.backup.manifest 2024-12-05T15:47:57,216 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733413657238/.backup.manifest 2024-12-05T15:47:57,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:47:57,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:47:57,425 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(116): writing 99 rows to test-1733413646382 2024-12-05T15:47:57,443 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(119): written 99 rows to test-1733413646382 2024-12-05T15:47:57,624 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(123): written 111 rows to test-1733413646382 to Mob enabled CF 2024-12-05T15:47:57,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17334136463821', row='row-t20', locateType=CURRENT is [region=ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:47:57,653 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(136): written 5 rows to ns2:test-17334136463821 2024-12-05T15:47:57,660 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$3(2313): Client=jenkins//172.17.0.3 split test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:47:57,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] assignment.SplitTableRegionProcedure(210): test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. splitKey isn't explicitly specified, will try to find a best split key from RS 5a29627a94df,34815,1733413637643 2024-12-05T15:47:57,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8 because midkey is the same as first or last row 2024-12-05T15:47:57,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f3/c7bc8ee8395f42e39c50f2005338ee85 because midkey is the same as first or last row 2024-12-05T15:47:57,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/09842ffb0751416eaaeffe3bff7aefa1 because midkey is the same as first or last row 2024-12-05T15:47:57,665 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.HRegion(2902): Flushing 24a988b9e3812c25fcebd4f4072c48f0 3/3 column families, dataSize=11.48 KB heapSize=27.79 KB 2024-12-05T15:47:57,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/f/91dfff1e2dee4ba59bbdb059b1d0570f is 59, key is row-test-1733413646382-1-10/f:q1/1733413677257/Put/seqid=0 2024-12-05T15:47:57,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741880_1056 (size=10796) 2024-12-05T15:47:58,090 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/f/91dfff1e2dee4ba59bbdb059b1d0570f 2024-12-05T15:47:58,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205070e4aa4b5934388ad16285a00c5ad73_24a988b9e3812c25fcebd4f4072c48f0 is 63, key is row-test-1733413646382-3-100/mob:q1/1733413677609/Put/seqid=0 2024-12-05T15:47:58,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741881_1057 (size=5591) 2024-12-05T15:47:58,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:47:58,527 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205070e4aa4b5934388ad16285a00c5ad73_24a988b9e3812c25fcebd4f4072c48f0 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/mobdir/data/default/test-1733413646382/d269731e9ae3c5ae370e1788a2e183c3/mob/d41d8cd98f00b204e9800998ecf8427e20241205070e4aa4b5934388ad16285a00c5ad73_24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:47:58,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/mob/d940ad880da148ee8bac344a248bd8c3, store: [table=test-1733413646382 family=mob region=24a988b9e3812c25fcebd4f4072c48f0] 2024-12-05T15:47:58,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/mob/d940ad880da148ee8bac344a248bd8c3 is 192, key is row-test-1733413646382-3-100/mob:q1/1733413677609/Put/seqid=0 2024-12-05T15:47:58,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741882_1058 (size=13760) 2024-12-05T15:47:58,936 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=6.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/mob/d940ad880da148ee8bac344a248bd8c3 2024-12-05T15:47:58,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/f/91dfff1e2dee4ba59bbdb059b1d0570f as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/91dfff1e2dee4ba59bbdb059b1d0570f 2024-12-05T15:47:58,961 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/91dfff1e2dee4ba59bbdb059b1d0570f, entries=99, sequenceid=334, filesize=10.5 K 2024-12-05T15:47:58,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/mob/d940ad880da148ee8bac344a248bd8c3 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/d940ad880da148ee8bac344a248bd8c3 2024-12-05T15:47:58,969 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/d940ad880da148ee8bac344a248bd8c3, entries=111, sequenceid=334, filesize=13.4 K 2024-12-05T15:47:58,970 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.48 KB/11754, heapSize ~27.51 KB/28168, currentSize=0 B/0 for 24a988b9e3812c25fcebd4f4072c48f0 in 1305ms, sequenceid=334, compaction requested=false 2024-12-05T15:47:58,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.HRegion(2603): Flush status journal for 24a988b9e3812c25fcebd4f4072c48f0: 2024-12-05T15:47:58,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/91dfff1e2dee4ba59bbdb059b1d0570f because midkey is the same as first or last row 2024-12-05T15:47:58,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f3/c7bc8ee8395f42e39c50f2005338ee85 because midkey is the same as first or last row 2024-12-05T15:47:58,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/d940ad880da148ee8bac344a248bd8c3 because midkey is the same as first or last row 2024-12-05T15:47:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=5a29627a94df,34815,1733413637643 2024-12-05T15:47:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T15:47:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.CallRunner(138): callId: 303 service: MasterService methodName: SplitRegion size: 100 connection: 172.17.0.3:55706 deadline: 1733413737659, exception=org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row 2024-12-05T15:47:58,978 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(147): region is not splittable, because java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:47:58,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: SPLIT_REGION, Table Name: default:test-1733413646382 failed with org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:47:58,978 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T15:47:58,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733413646382,, stopping at row=test-1733413646382 ,, for max=2147483647 with caching=100 2024-12-05T15:47:58,989 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733413646382,, stopping at row=test-1733413646382 ,, for max=2147483647 with caching=100 2024-12-05T15:47:58,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733413646382', row='', locateType=CURRENT is [region=test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0., hostname=5a29627a94df,34815,1733413637643, seqNum=107] 2024-12-05T15:47:59,000 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(152): split finished in =1346 2024-12-05T15:47:59,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:47:59,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:47:59,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:47:59,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:47:59,044 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-05T15:47:59,048 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733413679000 2024-12-05T15:47:59,054 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733413657238} 2024-12-05T15:47:59,054 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-12-05T15:47:59,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:47:59,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:47:59,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:47:59,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:47:59,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-05T15:47:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733413679108 (current time:1733413679108). 2024-12-05T15:47:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-05T15:47:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-05T15:47:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57202201, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:47:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:47:59,110 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:47:59,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:47:59,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:47:59,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c80dfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:59,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:47:59,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:47:59,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:59,113 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:47:59,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@797bfdb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:47:59,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:47:59,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:47:59,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:47:59,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:47:59,117 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117. 2024-12-05T15:47:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:47:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:47:59,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:47:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-05T15:47:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-05T15:47:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-12-05T15:47:59,121 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-05T15:47:59,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-05T15:47:59,122 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-05T15:47:59,124 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-05T15:47:59,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741883_1059 (size=52) 2024-12-05T15:47:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-05T15:47:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-05T15:47:59,533 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-05T15:47:59,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af}] 2024-12-05T15:47:59,536 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af 2024-12-05T15:47:59,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2024-12-05T15:47:59,689 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:47:59,690 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2902): Flushing f422dcfec85c6b145613ea93a68881af 2/2 column families, dataSize=1.44 KB heapSize=2.75 KB 2024-12-05T15:47:59,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/meta/e274103cce0e470c9f4fdd2b8874ffb0 is 158, key is trslm:hdfs://localhost:37233/backupUT\x00ns2:test-17334136463821/meta:log-roll-map/1733413676707/Put/seqid=0 2024-12-05T15:47:59,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741884_1060 (size=5914) 2024-12-05T15:47:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-05T15:48:00,116 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/meta/e274103cce0e470c9f4fdd2b8874ffb0 2024-12-05T15:48:00,127 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/session/d9b315e36bf644cdb0a8feb6197ddaf0 is 433, key is session:backup_1733413657238/session:context/1733413677172/Put/seqid=0 2024-12-05T15:48:00,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741885_1061 (size=5535) 2024-12-05T15:48:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-05T15:48:00,531 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/session/d9b315e36bf644cdb0a8feb6197ddaf0 2024-12-05T15:48:00,539 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/meta/e274103cce0e470c9f4fdd2b8874ffb0 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/e274103cce0e470c9f4fdd2b8874ffb0 2024-12-05T15:48:00,546 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/e274103cce0e470c9f4fdd2b8874ffb0, entries=6, sequenceid=17, filesize=5.8 K 2024-12-05T15:48:00,547 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/session/d9b315e36bf644cdb0a8feb6197ddaf0 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/d9b315e36bf644cdb0a8feb6197ddaf0 2024-12-05T15:48:00,554 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/d9b315e36bf644cdb0a8feb6197ddaf0, entries=2, sequenceid=17, filesize=5.4 K 2024-12-05T15:48:00,555 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3140): Finished flush of dataSize ~1.44 KB/1478, heapSize ~2.72 KB/2784, currentSize=0 B/0 for f422dcfec85c6b145613ea93a68881af in 866ms, sequenceid=17, compaction requested=false 2024-12-05T15:48:00,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2603): Flush status journal for f422dcfec85c6b145613ea93a68881af: 2024-12-05T15:48:00,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. for snapshot_backup_system completed. 2024-12-05T15:48:00,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af.' region-info for snapshot=snapshot_backup_system 2024-12-05T15:48:00,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-05T15:48:00,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/e274103cce0e470c9f4fdd2b8874ffb0] hfiles 2024-12-05T15:48:00,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/e274103cce0e470c9f4fdd2b8874ffb0 for snapshot=snapshot_backup_system 2024-12-05T15:48:00,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/ff33f5218d644a85986a494156767a9c, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/d9b315e36bf644cdb0a8feb6197ddaf0] hfiles 2024-12-05T15:48:00,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/ff33f5218d644a85986a494156767a9c for snapshot=snapshot_backup_system 2024-12-05T15:48:00,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/d9b315e36bf644cdb0a8feb6197ddaf0 for snapshot=snapshot_backup_system 2024-12-05T15:48:00,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741886_1062 (size=173) 2024-12-05T15:48:00,964 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:48:00,964 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-12-05T15:48:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-12-05T15:48:00,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region f422dcfec85c6b145613ea93a68881af 2024-12-05T15:48:00,967 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af 2024-12-05T15:48:00,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-05T15:48:00,974 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-05T15:48:00,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af in 1.4350 sec 2024-12-05T15:48:00,976 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-05T15:48:00,978 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-05T15:48:00,978 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-05T15:48:00,979 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-05T15:48:00,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741887_1063 (size=810) 2024-12-05T15:48:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-05T15:48:01,394 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-05T15:48:01,403 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-05T15:48:01,404 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_backup_system 2024-12-05T15:48:01,405 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-05T15:48:01,405 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-12-05T15:48:01,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2870 sec 2024-12-05T15:48:03,046 DEBUG [master/5a29627a94df:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733413646382 2024-12-05T15:48:03,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741888_1064 (size=950) 2024-12-05T15:48:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-05T15:48:03,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-05T15:48:03,278 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733413679000 started at 1733413683278. 2024-12-05T15:48:03,283 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733413679000 has been started. 2024-12-05T15:48:03,291 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [ns2:test-17334136463821, test-1733413646382] 2024-12-05T15:48:03,296 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1733413648077for backupID backup_1733413679000 2024-12-05T15:48:03,296 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-12-05T15:48:03,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-12-05T15:48:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-05T15:48:03,297 INFO [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-05T15:48:03,297 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-05T15:48:03,297 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-05T15:48:03,297 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-05T15:48:03,298 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-05T15:48:03,298 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-05T15:48:03,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-05T15:48:03,343 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,343 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-05T15:48:03,343 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-05T15:48:03,344 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,344 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-05T15:48:03,345 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-05T15:48:03,346 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-05T15:48:03,347 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-05T15:48:03,347 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-05T15:48:03,347 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-05T15:48:03,348 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-05T15:48:03,348 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-05T15:48:03,348 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-05T15:48:03,349 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-05T15:48:03,349 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-05T15:48:03,349 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-05T15:48:03,349 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '5a29627a94df,34815,1733413637643' joining acquired barrier for procedure (rolllog) in zk 2024-12-05T15:48:03,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,359 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:03,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-05T15:48:03,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-05T15:48:03,360 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-05T15:48:03,360 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-05T15:48:03,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-05T15:48:03,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-05T15:48:03,361 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:03,361 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,362 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-05T15:48:03,362 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '5a29627a94df,34815,1733413637643' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-05T15:48:03,362 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-05T15:48:03,362 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4a336f96[Count = 0] remaining members to acquire global barrier 2024-12-05T15:48:03,362 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:03,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:03,368 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:03,368 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:03,368 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-05T15:48:03,368 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,368 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-05T15:48:03,368 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-05T15:48:03,369 DEBUG [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,369 INFO [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733413664087 highest: 1733413664087 on 5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,369 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1:(num 1733413664068) roll requested 2024-12-05T15:48:03,383 DEBUG [regionserver/5a29627a94df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413683369, exclude list is [], retry=0 2024-12-05T15:48:03,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:48:03,388 INFO [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413664068 with entries=224, filesize=29.77 KB; new WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413683369 2024-12-05T15:48:03,388 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:48:03,388 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413664068 is not closed yet, will try archiving it next time 2024-12-05T15:48:03,388 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0:(num 1733413664087) roll requested 2024-12-05T15:48:03,389 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413648077 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413648077 2024-12-05T15:48:03,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741857_1033 (size=30490) 2024-12-05T15:48:03,401 DEBUG [regionserver/5a29627a94df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413683389, exclude list is [], retry=0 2024-12-05T15:48:03,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:48:03,405 INFO [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413664087 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413683389 2024-12-05T15:48:03,405 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:48:03,405 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413664087 is not closed yet, will try archiving it next time 2024-12-05T15:48:03,405 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5a29627a94df%2C34815%2C1733413637643.meta:.meta(num 1733413664106) roll requested 2024-12-05T15:48:03,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741858_1034 (size=1198) 2024-12-05T15:48:03,406 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413664087 not finished, retry = 0 2024-12-05T15:48:03,418 DEBUG [regionserver/5a29627a94df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413683405.meta, exclude list is [], retry=0 2024-12-05T15:48:03,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:48:03,422 INFO [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413664106.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413683405.meta 2024-12-05T15:48:03,422 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:48:03,422 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413664106.meta is not closed yet, will try archiving it next time 2024-12-05T15:48:03,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741859_1035 (size=93) 2024-12-05T15:48:03,423 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413664106.meta not finished, retry = 0 2024-12-05T15:48:03,429 DEBUG [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-12-05T15:48:03,429 INFO [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733413683389 on 5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:03,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:03,451 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-05T15:48:03,451 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '5a29627a94df,34815,1733413637643' in zk 2024-12-05T15:48:03,459 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-05T15:48:03,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,459 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-05T15:48:03,459 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-05T15:48:03,459 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,459 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-05T15:48:03,459 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-05T15:48:03,460 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-05T15:48:03,461 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-05T15:48:03,461 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:03,462 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,463 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-05T15:48:03,463 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/.tabledesc/.tableinfo.0000000002.908 2024-12-05T15:48:03,463 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/.tabledesc/.tableinfo.0000000003.950 2024-12-05T15:48:03,463 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:03,464 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,466 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '5a29627a94df,34815,1733413637643': 2024-12-05T15:48:03,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733413646382 in 422 msec 2024-12-05T15:48:03,466 INFO [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-05T15:48:03,466 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-05T15:48:03,466 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '5a29627a94df,34815,1733413637643' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-05T15:48:03,466 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-05T15:48:03,466 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-05T15:48:03,466 INFO [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-05T15:48:03,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-05T15:48:03,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-05T15:48:03,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-05T15:48:03,476 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-05T15:48:03,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-05T15:48:03,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-05T15:48:03,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-05T15:48:03,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-05T15:48:03,477 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-05T15:48:03,477 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-05T15:48:03,477 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-05T15:48:03,477 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,478 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:03,478 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-05T15:48:03,478 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-05T15:48:03,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:03,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,480 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-05T15:48:03,481 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:03,481 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-05T15:48:03,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-05T15:48:03,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-05T15:48:03,492 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-05T15:48:03,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:03,492 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-05T15:48:03,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-05T15:48:03,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:03,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:03,493 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-05T15:48:03,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-05T15:48:03,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-05T15:48:03,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-05T15:48:03,493 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-05T15:48:03,493 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-05T15:48:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-05T15:48:03,499 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {5a29627a94df:34815=1733413648077} newestTimestamps: {5a29627a94df:34815=1733413664087} 2024-12-05T15:48:03,502 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 5a29627a94df%2C34815%2C1733413637643.meta.1733413641984.meta 2024-12-05T15:48:03,502 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 5a29627a94df%2C34815%2C1733413637643.meta.1733413664106.meta 2024-12-05T15:48:03,502 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 5a29627a94df%2C34815%2C1733413637643.meta.1733413683405.meta 2024-12-05T15:48:03,503 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413664087 2024-12-05T15:48:03,503 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413683389 2024-12-05T15:48:03,503 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413664068 2024-12-05T15:48:03,503 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413683369 2024-12-05T15:48:03,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741892_1068 (size=364) 2024-12-05T15:48:03,525 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413664106.meta to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs/5a29627a94df%2C34815%2C1733413637643.meta.1733413664106.meta 2024-12-05T15:48:03,919 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17334136463821 target: hdfs://localhost:37233/backupUT/backup_1733413679000/ns2/test-17334136463821 descriptor: 'ns2:test-17334136463821', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:48:03,919 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-05T15:48:03,927 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17334136463821 2024-12-05T15:48:03,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741893_1069 (size=50) 2024-12-05T15:48:04,336 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17334136463821 2024-12-05T15:48:04,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741894_1070 (size=950) 2024-12-05T15:48:04,756 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1733413646382 target: hdfs://localhost:37233/backupUT/backup_1733413679000/default/test-1733413646382 descriptor: 'test-1733413646382', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:48:04,756 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-05T15:48:04,761 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1733413646382 2024-12-05T15:48:04,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741895_1071 (size=53) 2024-12-05T15:48:05,170 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1733413646382 2024-12-05T15:48:05,182 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:37233/backupUT/.tmp/backup_1733413679000 from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413664087;hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413664068 2024-12-05T15:48:05,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a00e2c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:05,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:48:05,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:48:05,185 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:48:05,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:48:05,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:48:05,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@644bae74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:05,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:48:05,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:48:05,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:05,186 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:48:05,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75330876, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:05,188 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:48:05,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:48:05,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@204349cd 2024-12-05T15:48:05,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:48:05,189 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:48:05,194 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-05T15:48:05,195 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17334136463821 2024-12-05T15:48:05,195 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17334136463821,, stopping at row=ns2:test-17334136463821 ,, for max=2147483647 with caching=100 2024-12-05T15:48:05,195 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:48:05,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:48:05,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:48:05,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:48:05,199 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17334136463821: ns2:test-17334136463821; 2024-12-05T15:48:05,199 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1733413646382 2024-12-05T15:48:05,200 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733413646382,, stopping at row=test-1733413646382 ,, for max=2147483647 with caching=100 2024-12-05T15:48:05,201 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1733413646382: test-1733413646382; 2024-12-05T15:48:05,202 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-12-05T15:48:05,202 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/partitions_8e618142-be82-4e1f-9506-1a0679410293 2024-12-05T15:48:05,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741896_1072 (size=192) 2024-12-05T15:48:05,631 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:05,632 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:05,632 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,404 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-18148842552260337801.jar 2024-12-05T15:48:06,405 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,405 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,405 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,406 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,406 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,406 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,406 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,407 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,407 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,407 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:48:06,407 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-05T15:48:06,408 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-05T15:48:06,408 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-05T15:48:06,408 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-05T15:48:06,408 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-05T15:48:06,408 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-05T15:48:06,409 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-05T15:48:06,409 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-05T15:48:06,409 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-05T15:48:06,409 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-05T15:48:06,410 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,411 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,411 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,411 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,411 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,412 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:06,412 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:06,412 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:test-17334136463821,default:test-1733413646382 2024-12-05T15:48:06,412 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:48:06,412 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:157) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:48:06,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:06,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:06,412 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:48:06,413 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:48:07,172 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-3693810984512577426.jar 2024-12-05T15:48:07,182 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:48:07,205 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413664087 for WAL files 2024-12-05T15:48:07,207 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413664087 2024-12-05T15:48:07,207 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413664068 for WAL files 2024-12-05T15:48:07,208 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413664068 2024-12-05T15:48:07,303 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local292032218_0003_4a8de8f8-1591-4276-b497-132fc1d7c885/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-05T15:48:07,303 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local292032218_0003_4a8de8f8-1591-4276-b497-132fc1d7c885/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-05T15:48:07,391 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413664068 for hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413664068 (-9223372036854775808:9223372036854775807) length:30490 2024-12-05T15:48:07,414 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-12-05T15:48:07,414 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-12-05T15:48:07,452 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413664087 for hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413664087 (-9223372036854775808:9223372036854775807) length:1198 2024-12-05T15:48:07,457 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-12-05T15:48:07,457 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-12-05T15:48:07,531 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1087129, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:07,531 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:48:07,532 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:48:07,532 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:48:07,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:48:07,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:48:07,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7f9dfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:07,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:48:07,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:48:07,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:07,534 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:48:07,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c628fab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:07,535 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:48:07,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:48:07,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:48:07,538 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:48:07,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17334136463821', row='row-t20', locateType=CURRENT is [region=ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:48:07,540 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-12-05T15:48:07,540 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:48:07,540 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:07,541 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:07,541 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:48:07,541 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-12-05T15:48:07,541 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5a29627a94df 2024-12-05T15:48:07,546 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/backupUT/.tmp/backup_1733413679000/_temporary/0/_temporary/attempt_local292032218_0003_r_000000_0/ns2/test-17334136463821/f/db00684815454cdb84701e3109b7dad4 is 38, key is row-t20/f:q1/1733413677638/Put/seqid=0 2024-12-05T15:48:07,548 WARN [Thread-950 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5a29627a94df:34815] Specified favored nodes: [5a29627a94df:34815] 2024-12-05T15:48:07,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741897_1073 (size=5274) 2024-12-05T15:48:07,986 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a448bba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:07,986 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:48:07,986 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:48:07,987 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:48:07,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:48:07,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:48:07,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3725d880, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:07,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:48:07,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:48:07,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:07,989 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:48:07,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bbd577, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:07,990 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:48:07,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:48:07,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:48:07,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:48:07,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733413646382', row='row-test-1733413646382-1-0', locateType=CURRENT is [region=test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0., hostname=5a29627a94df,34815,1733413637643, seqNum=107] 2024-12-05T15:48:07,995 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-12-05T15:48:07,995 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:48:07,995 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:07,995 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:07,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:48:07,995 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733413646382-1-0] 2024-12-05T15:48:07,995 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5a29627a94df 2024-12-05T15:48:08,008 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f23002f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:08,008 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:48:08,008 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:48:08,009 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:48:08,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:48:08,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:48:08,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708bbf1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:08,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:48:08,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:48:08,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:08,011 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:48:08,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@700527c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:08,012 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:48:08,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:48:08,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:48:08,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:48:08,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733413646382', row='row-test-1733413646382-3-0', locateType=CURRENT is [region=test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0., hostname=5a29627a94df,34815,1733413637643, seqNum=107] 2024-12-05T15:48:08,016 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-12-05T15:48:08,016 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:48:08,016 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:08,016 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:08,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:48:08,017 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733413646382-3-0] 2024-12-05T15:48:08,017 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5a29627a94df 2024-12-05T15:48:08,025 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/backupUT/.tmp/backup_1733413679000/_temporary/0/_temporary/attempt_local292032218_0003_r_000001_0/default/test-1733413646382/f/5b60b4c7f2fd4cf393a2af4d8ca00341 is 59, key is row-test-1733413646382-1-10/f:q1/1733413677257/Put/seqid=0 2024-12-05T15:48:08,027 WARN [Thread-962 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5a29627a94df:34815] Specified favored nodes: [5a29627a94df:34815] 2024-12-05T15:48:08,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741898_1074 (size=10769) 2024-12-05T15:48:08,433 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/backupUT/.tmp/backup_1733413679000/_temporary/0/_temporary/attempt_local292032218_0003_r_000001_0/default/test-1733413646382/mob/6f019982af04436381f8033e467a67f4 is 63, key is row-test-1733413646382-3-100/mob:q1/1733413677609/Put/seqid=0 2024-12-05T15:48:08,437 WARN [Thread-964 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5a29627a94df:34815] Specified favored nodes: [5a29627a94df:34815] 2024-12-05T15:48:08,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741899_1075 (size=11810) 2024-12-05T15:48:09,352 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:37233/backupUT 2024-12-05T15:48:09,352 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1733413679000 2024-12-05T15:48:09,352 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-12-05T15:48:09,369 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:37233/backupUT/.tmp/backup_1733413679000, hdfs://localhost:37233/backupUT] 2024-12-05T15:48:09,438 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:48:09,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741900_1076 (size=10769) 2024-12-05T15:48:10,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741901_1077 (size=5274) 2024-12-05T15:48:10,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741902_1078 (size=11810) 2024-12-05T15:48:11,039 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-12-05T15:48:11,043 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1733413679000 2024-12-05T15:48:11,044 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 27853 bytes copied." 2024-12-05T15:48:11,044 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local2095165901_0004 completed: true true 2024-12-05T15:48:11,046 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=3831860 FILE: Number of bytes written=6217936 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=231561 HDFS: Number of bytes written=2433146 HDFS: Number of read operations=646 HDFS: Number of large read operations=0 HDFS: Number of write operations=248 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=12 Map output records=0 Input split bytes=343 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=3004 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=27853 Bytes Copied=27853 Bytes Expected=27853 Files Copied=4 DIR_COPY=8 2024-12-05T15:48:11,046 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:37233/backupUT/.tmp/backup_1733413679000 to hdfs://localhost:37233/backupUT finished. 2024-12-05T15:48:11,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system_bulk', row='bulk:ns2:test-17334136463821:', locateType=CURRENT is [region=backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:48:11,061 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733413679000 2024-12-05T15:48:11,065 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733413657238} 2024-12-05T15:48:11,065 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-12-05T15:48:11,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741903_1079 (size=317) 2024-12-05T15:48:11,473 INFO [Time-limited test {}] impl.BackupManifest(504): Manifest file stored to hdfs://localhost:37233/backupUT/backup_1733413679000/.backup.manifest 2024-12-05T15:48:11,473 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1733413679000 finished: type=INCREMENTAL,tablelist=ns2:test-17334136463821;test-1733413646382,targetRootDir=hdfs://localhost:37233/backupUT,startts=1733413683278,completets=1733413691061,bytescopied=0 2024-12-05T15:48:11,473 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-05T15:48:11,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-12-05T15:48:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-05T15:48:11,487 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-05T15:48:11,489 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-05T15:48:11,493 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1733413679000 completed. 2024-12-05T15:48:11,493 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-12-05T15:48:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:11,513 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:37233/backupUT/backup_1733413679000/.backup.manifest 2024-12-05T15:48:11,515 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733413679000/.backup.manifest 2024-12-05T15:48:11,516 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:37233/backupUT/backup_1733413679000/.backup.manifest 2024-12-05T15:48:11,518 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733413679000/.backup.manifest 2024-12-05T15:48:11,520 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table test-1733413646382 from 'test-1733413646382', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1733413646382', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:48:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1733413646382 2024-12-05T15:48:11,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741904_1080 (size=908) 2024-12-05T15:48:11,939 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/.tabledesc/.tableinfo.0000000003.950 2024-12-05T15:48:11,939 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/.tabledesc/.tableinfo.0000000004.908 2024-12-05T15:48:11,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1733413646382}] 2024-12-05T15:48:11,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, REOPEN/MOVE}] 2024-12-05T15:48:11,949 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, REOPEN/MOVE 2024-12-05T15:48:11,949 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=24a988b9e3812c25fcebd4f4072c48f0, regionState=CLOSING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:48:11,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, REOPEN/MOVE because future has completed 2024-12-05T15:48:11,952 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T15:48:11,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:48:12,105 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(122): Close 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:12,105 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T15:48:12,105 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1722): Closing 24a988b9e3812c25fcebd4f4072c48f0, disabling compactions & flushes 2024-12-05T15:48:12,105 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1755): Closing region test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:48:12,105 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:48:12,105 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1843): Acquired close lock on test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. after waiting 0 ms 2024-12-05T15:48:12,105 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1853): Updates disabled for region test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:48:12,111 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=106 2024-12-05T15:48:12,111 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:48:12,111 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1973): Closed test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:48:12,111 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1676): Region close journal for 24a988b9e3812c25fcebd4f4072c48f0: Waiting for close lock at 1733413692105Running coprocessor pre-close hooks at 1733413692105Disabling compacts and flushes for region at 1733413692105Disabling writes for close at 1733413692105Writing region close event to WAL at 1733413692107 (+2 ms)Running coprocessor post-close hooks at 1733413692111 (+4 ms)Closed at 1733413692111 2024-12-05T15:48:12,112 WARN [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegionServer(3299): Not adding moved region record: 24a988b9e3812c25fcebd4f4072c48f0 to self. 2024-12-05T15:48:12,114 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(157): Closed 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:12,114 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=24a988b9e3812c25fcebd4f4072c48f0, regionState=CLOSED 2024-12-05T15:48:12,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:48:12,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-05T15:48:12,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0, server=5a29627a94df,34815,1733413637643 in 165 msec 2024-12-05T15:48:12,119 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, REOPEN/MOVE; state=CLOSED, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=true 2024-12-05T15:48:12,255 DEBUG [master/5a29627a94df:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733413646382 2024-12-05T15:48:12,255 DEBUG [master/5a29627a94df:0.Chore.1 {}] procedure.MasterProcedureScheduler(153): There is already a procedure running for table test-1733413646382, added pid=46, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733413646382 to waiting queue 2024-12-05T15:48:12,270 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=24a988b9e3812c25fcebd4f4072c48f0, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:48:12,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, REOPEN/MOVE because future has completed 2024-12-05T15:48:12,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:48:12,429 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:48:12,429 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 24a988b9e3812c25fcebd4f4072c48f0, NAME => 'test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0.', STARTKEY => '', ENDKEY => ''} 2024-12-05T15:48:12,430 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:48:12,430 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1733413646382 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:12,430 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:48:12,430 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:12,430 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:12,433 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:12,434 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24a988b9e3812c25fcebd4f4072c48f0 columnFamilyName f 2024-12-05T15:48:12,434 DEBUG [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:48:12,444 DEBUG [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8 2024-12-05T15:48:12,451 DEBUG [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/91dfff1e2dee4ba59bbdb059b1d0570f 2024-12-05T15:48:12,451 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.HStore(327): Store=24a988b9e3812c25fcebd4f4072c48f0/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:48:12,451 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:12,454 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24a988b9e3812c25fcebd4f4072c48f0 columnFamilyName f2 2024-12-05T15:48:12,454 DEBUG [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:48:12,454 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.HStore(327): Store=24a988b9e3812c25fcebd4f4072c48f0/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:48:12,455 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:12,456 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24a988b9e3812c25fcebd4f4072c48f0 columnFamilyName mob 2024-12-05T15:48:12,457 DEBUG [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:48:12,465 DEBUG [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/09842ffb0751416eaaeffe3bff7aefa1 2024-12-05T15:48:12,471 DEBUG [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/d940ad880da148ee8bac344a248bd8c3 2024-12-05T15:48:12,471 INFO [StoreOpener-24a988b9e3812c25fcebd4f4072c48f0-1 {}] regionserver.HStore(327): Store=24a988b9e3812c25fcebd4f4072c48f0/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:48:12,471 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:12,472 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:12,474 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:12,478 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:12,478 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:12,479 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1733413646382 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T15:48:12,481 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:12,482 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 24a988b9e3812c25fcebd4f4072c48f0; next sequenceid=338; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9420576800, jitterRate=-0.12264041602611542}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T15:48:12,482 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:12,484 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 24a988b9e3812c25fcebd4f4072c48f0: Running coprocessor pre-open hook at 1733413692431Writing region info on filesystem at 1733413692431Initializing all the Stores at 1733413692432 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413692432Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413692432Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413692432Cleaning up temporary data from old regions at 1733413692478 (+46 ms)Running coprocessor post-open hooks at 1733413692482 (+4 ms)Region opened successfully at 1733413692484 (+2 ms) 2024-12-05T15:48:12,485 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0., pid=47, masterSystemTime=1733413692425 2024-12-05T15:48:12,487 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:48:12,487 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:48:12,487 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=24a988b9e3812c25fcebd4f4072c48f0, regionState=OPEN, openSeqNum=338, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:48:12,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:48:12,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=44 2024-12-05T15:48:12,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=44, state=SUCCESS, hasLock=false; OpenRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0, server=5a29627a94df,34815,1733413637643 in 217 msec 2024-12-05T15:48:12,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-12-05T15:48:12,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1733413646382, region=24a988b9e3812c25fcebd4f4072c48f0, REOPEN/MOVE in 544 msec 2024-12-05T15:48:12,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-05T15:48:12,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1733413646382 in 550 msec 2024-12-05T15:48:12,499 DEBUG [PEWorker-3 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=test-1733413646382 2024-12-05T15:48:12,505 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f3/c7bc8ee8395f42e39c50f2005338ee85 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f3/c7bc8ee8395f42e39c50f2005338ee85 2024-12-05T15:48:12,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1733413646382 in 986 msec 2024-12-05T15:48:12,508 DEBUG [PEWorker-3 {}] procedure.MasterProcedureScheduler(373): pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1733413646382 completed, enqueue a new procedure pid=46, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733413646382 2024-12-05T15:48:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T15:48:12,509 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1733413646382 completed 2024-12-05T15:48:12,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741905_1081 (size=950) 2024-12-05T15:48:12,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:12,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:12,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:12,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:12,546 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-05T15:48:12,550 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733413692510 2024-12-05T15:48:12,553 DEBUG [Time-limited test {}] impl.TableBackupClient(330): Dependent incremental backup image: {BackupID=backup_1733413679000} 2024-12-05T15:48:12,553 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733413657238} 2024-12-05T15:48:12,553 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 2 ancestors for the current backup. 2024-12-05T15:48:12,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:12,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:12,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:12,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:12,594 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected Exception : Mismatch in column family descriptor for table: test-1733413646382 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException: Mismatch in column family descriptor for table: test-1733413646382 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException$ColumnFamilyMismatchExceptionBuilder.build(ColumnFamilyMismatchException.java:56) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.verifyCfCompatibility(IncrementalTableBackupClient.java:513) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:280) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.lambda$TestIncBackupRestore$0(TestIncrementalBackup.java:175) ~[test-classes/:?] at org.junit.Assert.assertThrows(Assert.java:1001) ~[junit-4.13.2.jar:4.13.2] at org.junit.Assert.assertThrows(Assert.java:981) ~[junit-4.13.2.jar:4.13.2] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:174) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T15:48:12,596 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1733413692510,startts=0,failedts=1733413692594,failedphase=REQUEST,failedmessage= Mismatch in column family descriptor for table: test-1733413646382 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'tru 2024-12-05T15:48:12,596 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-12-05T15:48:12,598 WARN [Time-limited test {}] impl.BackupSystemTable(1549): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2024-12-05T15:48:12,599 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-05T15:48:12,600 ERROR [Time-limited test {}] impl.BackupSystemTable(1578): Snapshot snapshot_backup_system does not exists 2024-12-05T15:48:12,600 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: REQUEST 2024-12-05T15:48:12,602 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-05T15:48:12,605 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1733413692510 failed. 2024-12-05T15:48:12,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:12,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:12,625 INFO [Time-limited test {}] impl.BackupManager(225): Full backup all the tables available in the cluster: [backup:system_bulk, test-1733413646382, ns2:test-17334136463821, ns3:test-17334136463822, ns4:test-17334136463823] 2024-12-05T15:48:12,625 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-05T15:48:12,630 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-05T15:48:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733413692630 (current time:1733413692630). 2024-12-05T15:48:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-05T15:48:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-05T15:48:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10c7e69e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:48:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:48:12,632 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:48:12,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:48:12,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:48:12,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a135eaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:12,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:48:12,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:48:12,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:12,634 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:48:12,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a25227b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:48:12,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:48:12,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:48:12,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:48:12,639 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117. 2024-12-05T15:48:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:48:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:12,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:48:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-05T15:48:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-05T15:48:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 48 2024-12-05T15:48:12,642 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-05T15:48:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T15:48:12,643 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-05T15:48:12,646 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-05T15:48:12,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741906_1082 (size=52) 2024-12-05T15:48:12,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T15:48:12,922 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/.tabledesc/.tableinfo.0000000004.908 2024-12-05T15:48:12,923 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/.tabledesc/.tableinfo.0000000005.950 2024-12-05T15:48:12,927 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733413646382 in 669 msec 2024-12-05T15:48:12,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T15:48:13,054 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-05T15:48:13,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af}] 2024-12-05T15:48:13,058 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=49, ppid=48, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af 2024-12-05T15:48:13,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=49 2024-12-05T15:48:13,210 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:48:13,211 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing f422dcfec85c6b145613ea93a68881af 2/2 column families, dataSize=2.61 KB heapSize=3.99 KB 2024-12-05T15:48:13,217 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/meta/f38659494b06472da7b8b4c06babbfa3 is 158, key is trslm:hdfs://localhost:37233/backupUT\x00ns2:test-17334136463821/meta:log-roll-map/1733413691051/Put/seqid=0 2024-12-05T15:48:13,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741907_1083 (size=5745) 2024-12-05T15:48:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T15:48:13,622 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=492 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/meta/f38659494b06472da7b8b4c06babbfa3 2024-12-05T15:48:13,631 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/session/03d3c4ccfd5d449f8c4a31f096312306 is 1245, key is session:backup_1733413692510/session:context/1733413692601/Put/seqid=0 2024-12-05T15:48:13,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741908_1084 (size=6682) 2024-12-05T15:48:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T15:48:14,038 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/session/03d3c4ccfd5d449f8c4a31f096312306 2024-12-05T15:48:14,051 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/meta/f38659494b06472da7b8b4c06babbfa3 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/f38659494b06472da7b8b4c06babbfa3 2024-12-05T15:48:14,057 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/f38659494b06472da7b8b4c06babbfa3, entries=4, sequenceid=31, filesize=5.6 K 2024-12-05T15:48:14,058 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/session/03d3c4ccfd5d449f8c4a31f096312306 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/03d3c4ccfd5d449f8c4a31f096312306 2024-12-05T15:48:14,064 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/03d3c4ccfd5d449f8c4a31f096312306, entries=3, sequenceid=31, filesize=6.5 K 2024-12-05T15:48:14,065 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~2.61 KB/2669, heapSize ~3.96 KB/4056, currentSize=0 B/0 for f422dcfec85c6b145613ea93a68881af in 855ms, sequenceid=31, compaction requested=true 2024-12-05T15:48:14,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for f422dcfec85c6b145613ea93a68881af: 2024-12-05T15:48:14,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. for snapshot_backup_system completed. 2024-12-05T15:48:14,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af.' region-info for snapshot=snapshot_backup_system 2024-12-05T15:48:14,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-05T15:48:14,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/e274103cce0e470c9f4fdd2b8874ffb0, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/f38659494b06472da7b8b4c06babbfa3] hfiles 2024-12-05T15:48:14,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/e274103cce0e470c9f4fdd2b8874ffb0 for snapshot=snapshot_backup_system 2024-12-05T15:48:14,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/f38659494b06472da7b8b4c06babbfa3 for snapshot=snapshot_backup_system 2024-12-05T15:48:14,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/ff33f5218d644a85986a494156767a9c, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/d9b315e36bf644cdb0a8feb6197ddaf0, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/03d3c4ccfd5d449f8c4a31f096312306] hfiles 2024-12-05T15:48:14,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/ff33f5218d644a85986a494156767a9c for snapshot=snapshot_backup_system 2024-12-05T15:48:14,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/d9b315e36bf644cdb0a8feb6197ddaf0 for snapshot=snapshot_backup_system 2024-12-05T15:48:14,066 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/03d3c4ccfd5d449f8c4a31f096312306 for snapshot=snapshot_backup_system 2024-12-05T15:48:14,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741909_1085 (size=251) 2024-12-05T15:48:14,472 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:48:14,473 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-05T15:48:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-05T15:48:14,474 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region f422dcfec85c6b145613ea93a68881af 2024-12-05T15:48:14,475 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=49, ppid=48, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af 2024-12-05T15:48:14,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-05T15:48:14,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af in 1.4220 sec 2024-12-05T15:48:14,482 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-05T15:48:14,483 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-05T15:48:14,484 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-05T15:48:14,484 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-05T15:48:14,485 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-05T15:48:14,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741910_1086 (size=888) 2024-12-05T15:48:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T15:48:14,901 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-05T15:48:14,913 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-05T15:48:14,914 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_backup_system 2024-12-05T15:48:14,915 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-05T15:48:14,915 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 48 2024-12-05T15:48:14,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2750 sec 2024-12-05T15:48:15,266 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T15:48:15,429 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region b7877185d3b50523892995990a3cd728, had cached 0 bytes from a total of 8413 2024-12-05T15:48:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T15:48:16,808 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-05T15:48:16,809 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733413692606 started at 1733413696808. 2024-12-05T15:48:16,813 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733413692606 has been started. 2024-12-05T15:48:16,818 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-12-05T15:48:16,819 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-12-05T15:48:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-05T15:48:16,820 INFO [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-05T15:48:16,820 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-05T15:48:16,820 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-05T15:48:16,820 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-05T15:48:16,822 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-05T15:48:16,822 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-05T15:48:16,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-05T15:48:16,859 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,859 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-05T15:48:16,859 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-05T15:48:16,860 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,860 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-05T15:48:16,860 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-05T15:48:16,862 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-05T15:48:16,863 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-05T15:48:16,863 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-05T15:48:16,863 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-05T15:48:16,863 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-05T15:48:16,864 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-05T15:48:16,864 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-05T15:48:16,864 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-05T15:48:16,864 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-05T15:48:16,864 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-05T15:48:16,864 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '5a29627a94df,34815,1733413637643' joining acquired barrier for procedure (rolllog) in zk 2024-12-05T15:48:16,875 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:16,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-05T15:48:16,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-05T15:48:16,876 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-05T15:48:16,876 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-05T15:48:16,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-05T15:48:16,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-05T15:48:16,877 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:16,877 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-05T15:48:16,878 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '5a29627a94df,34815,1733413637643' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-05T15:48:16,878 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@41e107e8[Count = 0] remaining members to acquire global barrier 2024-12-05T15:48:16,878 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-05T15:48:16,879 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:16,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:16,883 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:16,883 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:16,884 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-05T15:48:16,884 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,884 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-05T15:48:16,884 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-05T15:48:16,884 DEBUG [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,884 INFO [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733413683389 highest: 1733413683389 on 5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,884 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1:(num 1733413683369) roll requested 2024-12-05T15:48:16,901 DEBUG [regionserver/5a29627a94df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413696884, exclude list is [], retry=0 2024-12-05T15:48:16,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:48:16,906 INFO [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413683369 with entries=15, filesize=4.80 KB; new WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413696884 2024-12-05T15:48:16,906 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:48:16,906 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413683369 is not closed yet, will try archiving it next time 2024-12-05T15:48:16,906 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0:(num 1733413683389) roll requested 2024-12-05T15:48:16,906 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413664068 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413664068 2024-12-05T15:48:16,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741889_1065 (size=4927) 2024-12-05T15:48:16,907 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413683369 not finished, retry = 0 2024-12-05T15:48:16,922 DEBUG [regionserver/5a29627a94df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413696906, exclude list is [], retry=0 2024-12-05T15:48:16,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:48:16,926 INFO [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413683389 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413696906 2024-12-05T15:48:16,926 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:48:16,926 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413683389 is not closed yet, will try archiving it next time 2024-12-05T15:48:16,926 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5a29627a94df%2C34815%2C1733413637643.meta:.meta(num 1733413683405) roll requested 2024-12-05T15:48:16,927 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413683389 not finished, retry = 0 2024-12-05T15:48:16,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741890_1066 (size=93) 2024-12-05T15:48:16,937 DEBUG [regionserver/5a29627a94df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413696926.meta, exclude list is [], retry=0 2024-12-05T15:48:16,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:48:16,940 INFO [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413683405.meta with entries=4, filesize=1.85 KB; new WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413696926.meta 2024-12-05T15:48:16,940 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:48:16,940 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413683405.meta is not closed yet, will try archiving it next time 2024-12-05T15:48:16,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741891_1067 (size=1903) 2024-12-05T15:48:16,942 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413683405.meta not finished, retry = 0 2024-12-05T15:48:16,945 DEBUG [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-12-05T15:48:16,945 INFO [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733413696906 on 5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:16,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:16,963 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-05T15:48:16,963 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '5a29627a94df,34815,1733413637643' in zk 2024-12-05T15:48:16,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,973 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-05T15:48:16,973 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-05T15:48:16,973 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-05T15:48:16,973 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,974 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-05T15:48:16,974 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-05T15:48:16,974 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-05T15:48:16,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-05T15:48:16,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:16,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-05T15:48:16,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:16,978 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,979 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '5a29627a94df,34815,1733413637643': 2024-12-05T15:48:16,979 INFO [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-05T15:48:16,979 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '5a29627a94df,34815,1733413637643' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-05T15:48:16,979 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-05T15:48:16,980 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-05T15:48:16,980 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-05T15:48:16,980 INFO [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-05T15:48:16,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-05T15:48:16,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-05T15:48:16,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-05T15:48:16,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-05T15:48:16,990 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-05T15:48:16,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-05T15:48:16,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-05T15:48:16,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-05T15:48:16,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-05T15:48:16,991 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-05T15:48:16,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-05T15:48:16,991 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:16,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-05T15:48:16,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-05T15:48:16,992 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:16,992 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,992 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:16,993 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-05T15:48:16,993 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:16,993 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:48:17,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:17,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-05T15:48:17,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-05T15:48:17,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-05T15:48:17,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:17,000 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-05T15:48:17,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:17,000 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-05T15:48:17,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:17,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-05T15:48:17,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-05T15:48:17,000 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-05T15:48:17,000 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-05T15:48:17,000 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-05T15:48:17,001 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2024-12-05T15:48:17,001 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-05T15:48:17,001 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-05T15:48:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-05T15:48:17,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733413697009_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-12-05T15:48:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733413697010 (current time:1733413697010). 2024-12-05T15:48:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-05T15:48:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733413697009_backup_system_bulk VERSION not specified, setting to 2 2024-12-05T15:48:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6afd5e2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:48:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:48:17,011 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:48:17,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:48:17,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:48:17,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7037bacc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:17,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:48:17,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:48:17,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:17,013 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:48:17,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dcb6cba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:48:17,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:48:17,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:48:17,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:48:17,017 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117. 2024-12-05T15:48:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:48:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:17,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:48:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-05T15:48:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733413697009_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-12-05T15:48:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733413697009_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-12-05T15:48:17,019 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733413697009_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-05T15:48:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T15:48:17,020 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733413697009_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-05T15:48:17,022 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733413697009_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-05T15:48:17,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741914_1090 (size=76) 2024-12-05T15:48:17,028 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413683389 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413683389 2024-12-05T15:48:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T15:48:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T15:48:17,431 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733413697009_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-05T15:48:17,431 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure eb162556a9ba10393a1061e0fd710298}] 2024-12-05T15:48:17,434 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure eb162556a9ba10393a1061e0fd710298 2024-12-05T15:48:17,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2024-12-05T15:48:17,588 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. 2024-12-05T15:48:17,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for eb162556a9ba10393a1061e0fd710298: 2024-12-05T15:48:17,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. for snapshot_1733413697009_backup_system_bulk completed. 2024-12-05T15:48:17,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(241): Storing 'backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298.' region-info for snapshot=snapshot_1733413697009_backup_system_bulk 2024-12-05T15:48:17,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-05T15:48:17,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-05T15:48:17,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-05T15:48:17,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741915_1091 (size=61) 2024-12-05T15:48:17,640 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-12-05T15:48:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T15:48:18,001 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. 2024-12-05T15:48:18,002 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-05T15:48:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-05T15:48:18,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733413697009_backup_system_bulk on region eb162556a9ba10393a1061e0fd710298 2024-12-05T15:48:18,004 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure eb162556a9ba10393a1061e0fd710298 2024-12-05T15:48:18,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-05T15:48:18,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; SnapshotRegionProcedure eb162556a9ba10393a1061e0fd710298 in 575 msec 2024-12-05T15:48:18,009 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733413697009_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-05T15:48:18,010 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733413697009_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-05T15:48:18,010 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733413697009_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-05T15:48:18,011 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733413697009_backup_system_bulk 2024-12-05T15:48:18,012 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_1733413697009_backup_system_bulk 2024-12-05T15:48:18,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741916_1092 (size=702) 2024-12-05T15:48:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T15:48:18,429 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733413697009_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-05T15:48:18,439 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733413697009_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-05T15:48:18,439 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_1733413697009_backup_system_bulk to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_1733413697009_backup_system_bulk 2024-12-05T15:48:18,441 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733413697009_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-05T15:48:18,441 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733413697009_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-12-05T15:48:18,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733413697009_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } in 1.4230 sec 2024-12-05T15:48:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T15:48:19,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system_bulk completed 2024-12-05T15:48:19,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733413699177_ns3_test-17334136463822 table=ns3:test-17334136463822 type=FLUSH ttl=0 } 2024-12-05T15:48:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733413699178 (current time:1733413699178). 2024-12-05T15:48:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-05T15:48:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733413699177_ns3_test-17334136463822 VERSION not specified, setting to 2 2024-12-05T15:48:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@163cc985, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:48:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:48:19,180 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:48:19,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:48:19,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:48:19,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6812ac4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:19,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:48:19,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:48:19,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:19,182 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:48:19,183 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e038109, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:48:19,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:48:19,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:48:19,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:48:19,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117. 2024-12-05T15:48:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:48:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:19,187 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:48:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-05T15:48:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733413699177_ns3_test-17334136463822 table=ns3:test-17334136463822 type=FLUSH ttl=0 } 2024-12-05T15:48:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733413699177_ns3_test-17334136463822 table=ns3:test-17334136463822 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-12-05T15:48:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T15:48:19,190 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733413699177_ns3_test-17334136463822 table=ns3:test-17334136463822 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-05T15:48:19,192 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733413699177_ns3_test-17334136463822 table=ns3:test-17334136463822 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-05T15:48:19,194 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733413699177_ns3_test-17334136463822 table=ns3:test-17334136463822 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-05T15:48:19,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741917_1093 (size=86) 2024-12-05T15:48:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T15:48:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T15:48:19,604 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733413699177_ns3_test-17334136463822 table=ns3:test-17334136463822 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-05T15:48:19,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 750a447b5bb19a7405a6a02a816d1329}] 2024-12-05T15:48:19,607 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 750a447b5bb19a7405a6a02a816d1329 2024-12-05T15:48:19,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2024-12-05T15:48:19,761 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. 2024-12-05T15:48:19,761 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 750a447b5bb19a7405a6a02a816d1329: 2024-12-05T15:48:19,761 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. for snapshot_1733413699177_ns3_test-17334136463822 completed. 2024-12-05T15:48:19,762 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(241): Storing 'ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329.' region-info for snapshot=snapshot_1733413699177_ns3_test-17334136463822 2024-12-05T15:48:19,762 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-05T15:48:19,762 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-05T15:48:19,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741918_1094 (size=52) 2024-12-05T15:48:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T15:48:20,174 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. 2024-12-05T15:48:20,174 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-05T15:48:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-05T15:48:20,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733413699177_ns3_test-17334136463822 on region 750a447b5bb19a7405a6a02a816d1329 2024-12-05T15:48:20,175 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 750a447b5bb19a7405a6a02a816d1329 2024-12-05T15:48:20,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-05T15:48:20,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 750a447b5bb19a7405a6a02a816d1329 in 571 msec 2024-12-05T15:48:20,177 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733413699177_ns3_test-17334136463822 table=ns3:test-17334136463822 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-05T15:48:20,178 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733413699177_ns3_test-17334136463822 table=ns3:test-17334136463822 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-05T15:48:20,179 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733413699177_ns3_test-17334136463822 table=ns3:test-17334136463822 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-05T15:48:20,179 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733413699177_ns3_test-17334136463822 2024-12-05T15:48:20,179 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_1733413699177_ns3_test-17334136463822 2024-12-05T15:48:20,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741919_1095 (size=443) 2024-12-05T15:48:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T15:48:20,594 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733413699177_ns3_test-17334136463822 table=ns3:test-17334136463822 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-05T15:48:20,601 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733413699177_ns3_test-17334136463822 table=ns3:test-17334136463822 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-05T15:48:20,602 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_1733413699177_ns3_test-17334136463822 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_1733413699177_ns3_test-17334136463822 2024-12-05T15:48:20,603 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733413699177_ns3_test-17334136463822 table=ns3:test-17334136463822 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-05T15:48:20,604 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733413699177_ns3_test-17334136463822 table=ns3:test-17334136463822 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-12-05T15:48:20,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733413699177_ns3_test-17334136463822 table=ns3:test-17334136463822 type=FLUSH ttl=0 } in 1.4160 sec 2024-12-05T15:48:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T15:48:21,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns3:test-17334136463822 completed 2024-12-05T15:48:21,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733413701337_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } 2024-12-05T15:48:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733413701338 (current time:1733413701338). 2024-12-05T15:48:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-05T15:48:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733413701337_ns2_test-17334136463821 VERSION not specified, setting to 2 2024-12-05T15:48:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f8e68b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:48:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:48:21,342 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:48:21,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:48:21,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:48:21,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@838c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:21,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:48:21,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:48:21,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:21,345 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:48:21,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53564e94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:21,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:48:21,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:48:21,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:48:21,349 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:48:21,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117. 2024-12-05T15:48:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:48:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:21,351 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:48:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-05T15:48:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733413701337_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } 2024-12-05T15:48:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733413701337_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-12-05T15:48:21,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T15:48:21,354 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733413701337_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-05T15:48:21,355 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733413701337_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-05T15:48:21,357 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733413701337_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-05T15:48:21,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741920_1096 (size=86) 2024-12-05T15:48:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T15:48:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T15:48:21,767 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733413701337_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-05T15:48:21,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure b7877185d3b50523892995990a3cd728}] 2024-12-05T15:48:21,769 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure b7877185d3b50523892995990a3cd728 2024-12-05T15:48:21,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2024-12-05T15:48:21,923 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:48:21,924 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing b7877185d3b50523892995990a3cd728 1/1 column families, dataSize=170 B heapSize=816 B 2024-12-05T15:48:21,933 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/.tmp/f/d97c997a6e4d4d18b6b69960413bfcdd is 38, key is row-t20/f:q1/1733413677638/Put/seqid=0 2024-12-05T15:48:21,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741921_1097 (size=5134) 2024-12-05T15:48:21,942 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/.tmp/f/d97c997a6e4d4d18b6b69960413bfcdd 2024-12-05T15:48:21,949 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/.tmp/f/d97c997a6e4d4d18b6b69960413bfcdd as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/d97c997a6e4d4d18b6b69960413bfcdd 2024-12-05T15:48:21,956 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/d97c997a6e4d4d18b6b69960413bfcdd, entries=5, sequenceid=111, filesize=5.0 K 2024-12-05T15:48:21,957 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for b7877185d3b50523892995990a3cd728 in 33ms, sequenceid=111, compaction requested=false 2024-12-05T15:48:21,958 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for b7877185d3b50523892995990a3cd728: 2024-12-05T15:48:21,958 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. for snapshot_1733413701337_ns2_test-17334136463821 completed. 2024-12-05T15:48:21,958 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728.' region-info for snapshot=snapshot_1733413701337_ns2_test-17334136463821 2024-12-05T15:48:21,958 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-05T15:48:21,958 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/1159a028de3844c4b30999226eb32483, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/d97c997a6e4d4d18b6b69960413bfcdd] hfiles 2024-12-05T15:48:21,958 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/1159a028de3844c4b30999226eb32483 for snapshot=snapshot_1733413701337_ns2_test-17334136463821 2024-12-05T15:48:21,958 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/d97c997a6e4d4d18b6b69960413bfcdd for snapshot=snapshot_1733413701337_ns2_test-17334136463821 2024-12-05T15:48:21,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741922_1098 (size=130) 2024-12-05T15:48:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T15:48:22,366 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:48:22,366 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-05T15:48:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-05T15:48:22,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733413701337_ns2_test-17334136463821 on region b7877185d3b50523892995990a3cd728 2024-12-05T15:48:22,369 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure b7877185d3b50523892995990a3cd728 2024-12-05T15:48:22,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-05T15:48:22,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; SnapshotRegionProcedure b7877185d3b50523892995990a3cd728 in 604 msec 2024-12-05T15:48:22,377 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733413701337_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-05T15:48:22,379 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733413701337_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-05T15:48:22,379 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733413701337_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-05T15:48:22,379 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733413701337_ns2_test-17334136463821 2024-12-05T15:48:22,380 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_1733413701337_ns2_test-17334136463821 2024-12-05T15:48:22,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741923_1099 (size=496) 2024-12-05T15:48:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T15:48:22,795 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733413701337_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-05T15:48:22,806 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733413701337_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-05T15:48:22,807 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_1733413701337_ns2_test-17334136463821 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_1733413701337_ns2_test-17334136463821 2024-12-05T15:48:22,809 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733413701337_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-05T15:48:22,809 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733413701337_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-12-05T15:48:22,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733413701337_ns2_test-17334136463821 table=ns2:test-17334136463821 type=FLUSH ttl=0 } in 1.4580 sec 2024-12-05T15:48:22,885 INFO [master/5a29627a94df:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-05T15:48:22,885 INFO [master/5a29627a94df:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-05T15:48:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T15:48:23,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17334136463821 completed 2024-12-05T15:48:23,499 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733413703498_ns4_test-17334136463823 table=ns4:test-17334136463823 type=FLUSH ttl=0 } 2024-12-05T15:48:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733413703500 (current time:1733413703500). 2024-12-05T15:48:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-05T15:48:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733413703498_ns4_test-17334136463823 VERSION not specified, setting to 2 2024-12-05T15:48:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c7440c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:48:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:48:23,504 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:48:23,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:48:23,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:48:23,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33d64e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:23,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:48:23,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:48:23,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:23,509 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:48:23,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30fcebca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:48:23,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:48:23,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:48:23,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:48:23,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117. 2024-12-05T15:48:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:48:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:23,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:48:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-05T15:48:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733413703498_ns4_test-17334136463823 table=ns4:test-17334136463823 type=FLUSH ttl=0 } 2024-12-05T15:48:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733413703498_ns4_test-17334136463823 table=ns4:test-17334136463823 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-12-05T15:48:23,517 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733413703498_ns4_test-17334136463823 table=ns4:test-17334136463823 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-05T15:48:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T15:48:23,518 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733413703498_ns4_test-17334136463823 table=ns4:test-17334136463823 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-05T15:48:23,522 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733413703498_ns4_test-17334136463823 table=ns4:test-17334136463823 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-05T15:48:23,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741924_1100 (size=86) 2024-12-05T15:48:23,569 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region f422dcfec85c6b145613ea93a68881af, had cached 0 bytes from a total of 28925 2024-12-05T15:48:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T15:48:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T15:48:23,932 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733413703498_ns4_test-17334136463823 table=ns4:test-17334136463823 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-05T15:48:23,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d2c0907dfdcc66e63e68a2757d1d51ef}] 2024-12-05T15:48:23,934 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d2c0907dfdcc66e63e68a2757d1d51ef 2024-12-05T15:48:24,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2024-12-05T15:48:24,089 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. 2024-12-05T15:48:24,089 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for d2c0907dfdcc66e63e68a2757d1d51ef: 2024-12-05T15:48:24,089 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. for snapshot_1733413703498_ns4_test-17334136463823 completed. 2024-12-05T15:48:24,090 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(241): Storing 'ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef.' region-info for snapshot=snapshot_1733413703498_ns4_test-17334136463823 2024-12-05T15:48:24,090 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-05T15:48:24,090 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-05T15:48:24,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741925_1101 (size=52) 2024-12-05T15:48:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T15:48:24,504 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. 2024-12-05T15:48:24,505 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-05T15:48:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-05T15:48:24,506 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733413703498_ns4_test-17334136463823 on region d2c0907dfdcc66e63e68a2757d1d51ef 2024-12-05T15:48:24,507 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d2c0907dfdcc66e63e68a2757d1d51ef 2024-12-05T15:48:24,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-05T15:48:24,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; SnapshotRegionProcedure d2c0907dfdcc66e63e68a2757d1d51ef in 577 msec 2024-12-05T15:48:24,514 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733413703498_ns4_test-17334136463823 table=ns4:test-17334136463823 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-05T15:48:24,516 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733413703498_ns4_test-17334136463823 table=ns4:test-17334136463823 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-05T15:48:24,517 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733413703498_ns4_test-17334136463823 table=ns4:test-17334136463823 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-05T15:48:24,517 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733413703498_ns4_test-17334136463823 2024-12-05T15:48:24,518 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_1733413703498_ns4_test-17334136463823 2024-12-05T15:48:24,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741926_1102 (size=443) 2024-12-05T15:48:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T15:48:24,932 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733413703498_ns4_test-17334136463823 table=ns4:test-17334136463823 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-05T15:48:24,942 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733413703498_ns4_test-17334136463823 table=ns4:test-17334136463823 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-05T15:48:24,942 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_1733413703498_ns4_test-17334136463823 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_1733413703498_ns4_test-17334136463823 2024-12-05T15:48:24,944 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733413703498_ns4_test-17334136463823 table=ns4:test-17334136463823 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-05T15:48:24,944 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733413703498_ns4_test-17334136463823 table=ns4:test-17334136463823 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-12-05T15:48:24,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733413703498_ns4_test-17334136463823 table=ns4:test-17334136463823 type=FLUSH ttl=0 } in 1.4290 sec 2024-12-05T15:48:25,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T15:48:25,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns4:test-17334136463823 completed 2024-12-05T15:48:25,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733413705666_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } 2024-12-05T15:48:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733413705667 (current time:1733413705667). 2024-12-05T15:48:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-05T15:48:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733413705666_default_test-1733413646382 VERSION not specified, setting to 2 2024-12-05T15:48:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60399105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:48:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:48:25,670 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:48:25,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:48:25,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:48:25,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138178af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:25,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:48:25,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:48:25,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:25,673 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:48:25,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a3b3e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:48:25,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:48:25,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:48:25,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:48:25,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117. 2024-12-05T15:48:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:48:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:25,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:48:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-05T15:48:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733413705666_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } 2024-12-05T15:48:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733413705666_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-12-05T15:48:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T15:48:25,683 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733413705666_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-05T15:48:25,685 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733413705666_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-05T15:48:25,689 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733413705666_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-05T15:48:25,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741927_1103 (size=84) 2024-12-05T15:48:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T15:48:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T15:48:26,100 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733413705666_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-05T15:48:26,100 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0}] 2024-12-05T15:48:26,102 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:26,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2024-12-05T15:48:26,256 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:48:26,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 24a988b9e3812c25fcebd4f4072c48f0: 2024-12-05T15:48:26,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. for snapshot_1733413705666_default_test-1733413646382 completed. 2024-12-05T15:48:26,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(241): Storing 'test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0.' region-info for snapshot=snapshot_1733413705666_default_test-1733413646382 2024-12-05T15:48:26,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-05T15:48:26,258 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/91dfff1e2dee4ba59bbdb059b1d0570f] hfiles 2024-12-05T15:48:26,258 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8 for snapshot=snapshot_1733413705666_default_test-1733413646382 2024-12-05T15:48:26,258 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/91dfff1e2dee4ba59bbdb059b1d0570f for snapshot=snapshot_1733413705666_default_test-1733413646382 2024-12-05T15:48:26,258 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-05T15:48:26,258 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/09842ffb0751416eaaeffe3bff7aefa1, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/d940ad880da148ee8bac344a248bd8c3] hfiles 2024-12-05T15:48:26,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/09842ffb0751416eaaeffe3bff7aefa1 for snapshot=snapshot_1733413705666_default_test-1733413646382 2024-12-05T15:48:26,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/d940ad880da148ee8bac344a248bd8c3 for snapshot=snapshot_1733413705666_default_test-1733413646382 2024-12-05T15:48:26,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741928_1104 (size=224) 2024-12-05T15:48:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T15:48:26,668 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:48:26,669 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-05T15:48:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-05T15:48:26,669 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733413705666_default_test-1733413646382 on region 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:26,670 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:26,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-05T15:48:26,673 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733413705666_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-05T15:48:26,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 24a988b9e3812c25fcebd4f4072c48f0 in 571 msec 2024-12-05T15:48:26,674 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733413705666_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-05T15:48:26,676 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-12-05T15:48:26,676 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-12-05T15:48:26,676 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:48:26,677 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-12-05T15:48:26,677 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:48:26,677 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f2 2024-12-05T15:48:26,678 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:48:26,679 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/mobdir/data/default/test-1733413646382/d269731e9ae3c5ae370e1788a2e183c3/mob/d41d8cd98f00b204e9800998ecf8427e20241205070e4aa4b5934388ad16285a00c5ad73_24a988b9e3812c25fcebd4f4072c48f0] hfiles 2024-12-05T15:48:26,679 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/1): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/mobdir/data/default/test-1733413646382/d269731e9ae3c5ae370e1788a2e183c3/mob/d41d8cd98f00b204e9800998ecf8427e20241205070e4aa4b5934388ad16285a00c5ad73_24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:26,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741929_1105 (size=168) 2024-12-05T15:48:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T15:48:27,087 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733413705666_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-05T15:48:27,087 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733413705666_default_test-1733413646382 2024-12-05T15:48:27,088 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_1733413705666_default_test-1733413646382 2024-12-05T15:48:27,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741930_1106 (size=1347) 2024-12-05T15:48:27,498 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733413705666_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-05T15:48:27,506 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733413705666_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-05T15:48:27,507 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_1733413705666_default_test-1733413646382 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_1733413705666_default_test-1733413646382 2024-12-05T15:48:27,508 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733413705666_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-05T15:48:27,508 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733413705666_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-12-05T15:48:27,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733413705666_default_test-1733413646382 table=test-1733413646382 type=FLUSH ttl=0 } in 1.8270 sec 2024-12-05T15:48:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T15:48:27,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1733413646382 completed 2024-12-05T15:48:27,828 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1733413692606 2024-12-05T15:48:27,828 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-12-05T15:48:27,828 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 5 snapshots to be copied. 2024-12-05T15:48:27,829 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733413692606_backup:system_bulk 2024-12-05T15:48:27,829 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733413697009_backup_system_bulk to hdfs://localhost:37233/backupUT/backup_1733413692606/backup/system_bulk/ 2024-12-05T15:48:27,829 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-05T15:48:27,860 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:37233, inputRoot=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:48:27,860 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_51721194_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37233/backupUT/backup_1733413692606/backup/system_bulk, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37233/backupUT/backup_1733413692606/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1733413697009_backup_system_bulk 2024-12-05T15:48:27,862 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-05T15:48:27,866 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_1733413697009_backup_system_bulk to hdfs://localhost:37233/backupUT/backup_1733413692606/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1733413697009_backup_system_bulk 2024-12-05T15:48:27,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741932_1108 (size=76) 2024-12-05T15:48:27,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741931_1107 (size=702) 2024-12-05T15:48:28,289 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:28,290 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:28,291 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:29,054 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-17152693909256782377.jar 2024-12-05T15:48:29,054 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:29,055 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:29,055 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:29,055 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:29,055 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:29,055 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:29,056 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:29,056 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:29,056 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:29,056 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:48:29,057 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-05T15:48:29,057 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-05T15:48:29,057 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-05T15:48:29,057 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-05T15:48:29,057 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-05T15:48:29,058 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-05T15:48:29,058 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-05T15:48:29,058 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-05T15:48:29,058 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-05T15:48:29,058 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-05T15:48:29,059 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:29,059 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:29,059 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:29,059 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:29,060 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:29,060 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:29,060 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:29,093 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:48:29,115 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733413697009_backup_system_bulk' hfile list 2024-12-05T15:48:29,196 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local980735586_0005_576c65d8-5abb-4c4e-af2a-ccaaa2f1c67f/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-05T15:48:29,196 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local980735586_0005_576c65d8-5abb-4c4e-af2a-ccaaa2f1c67f/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-05T15:48:30,230 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-05T15:48:30,233 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-05T15:48:30,240 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733413697009_backup_system_bulk 2024-12-05T15:48:30,241 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733413697009_backup_system_bulk finished. 2024-12-05T15:48:30,241 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733413692606_ns3:test-17334136463822 2024-12-05T15:48:30,241 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733413699177_ns3_test-17334136463822 to hdfs://localhost:37233/backupUT/backup_1733413692606/ns3/test-17334136463822/ 2024-12-05T15:48:30,241 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-05T15:48:30,266 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:37233, inputRoot=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:48:30,266 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_51721194_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37233/backupUT/backup_1733413692606/ns3/test-17334136463822, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37233/backupUT/backup_1733413692606/ns3/test-17334136463822/.hbase-snapshot/.tmp/snapshot_1733413699177_ns3_test-17334136463822 2024-12-05T15:48:30,268 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-05T15:48:30,272 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_1733413699177_ns3_test-17334136463822 to hdfs://localhost:37233/backupUT/backup_1733413692606/ns3/test-17334136463822/.hbase-snapshot/.tmp/snapshot_1733413699177_ns3_test-17334136463822 2024-12-05T15:48:30,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741933_1109 (size=86) 2024-12-05T15:48:30,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741934_1110 (size=443) 2024-12-05T15:48:30,688 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:30,688 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:30,689 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:31,450 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-14285543936841879850.jar 2024-12-05T15:48:31,450 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:31,451 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:31,451 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:31,451 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:31,451 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:31,451 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:31,451 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:31,452 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:31,452 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:31,452 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:48:31,452 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-05T15:48:31,452 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-05T15:48:31,452 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-05T15:48:31,453 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-05T15:48:31,453 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-05T15:48:31,453 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-05T15:48:31,453 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-05T15:48:31,453 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-05T15:48:31,453 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-05T15:48:31,454 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-05T15:48:31,454 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:31,454 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:31,454 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:31,454 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:31,455 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:31,455 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:31,455 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:31,485 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:48:31,508 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733413699177_ns3_test-17334136463822' hfile list 2024-12-05T15:48:31,594 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local2030431044_0006_f48796cd-2e79-4fca-811d-e67f7656a2ec/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-05T15:48:31,594 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local2030431044_0006_f48796cd-2e79-4fca-811d-e67f7656a2ec/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-05T15:48:32,630 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-05T15:48:32,632 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-05T15:48:32,644 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733413699177_ns3_test-17334136463822 2024-12-05T15:48:32,644 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733413699177_ns3_test-17334136463822 finished. 2024-12-05T15:48:32,645 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733413692606_ns2:test-17334136463821 2024-12-05T15:48:32,645 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733413701337_ns2_test-17334136463821 to hdfs://localhost:37233/backupUT/backup_1733413692606/ns2/test-17334136463821/ 2024-12-05T15:48:32,645 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-05T15:48:32,671 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:37233, inputRoot=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:48:32,671 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_51721194_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37233/backupUT/backup_1733413692606/ns2/test-17334136463821, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37233/backupUT/backup_1733413692606/ns2/test-17334136463821/.hbase-snapshot/.tmp/snapshot_1733413701337_ns2_test-17334136463821 2024-12-05T15:48:32,673 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-05T15:48:32,679 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_1733413701337_ns2_test-17334136463821 to hdfs://localhost:37233/backupUT/backup_1733413692606/ns2/test-17334136463821/.hbase-snapshot/.tmp/snapshot_1733413701337_ns2_test-17334136463821 2024-12-05T15:48:32,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741936_1112 (size=496) 2024-12-05T15:48:32,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741935_1111 (size=86) 2024-12-05T15:48:33,097 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:33,098 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:33,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:33,865 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-12062955653476085251.jar 2024-12-05T15:48:33,866 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:33,866 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:33,866 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:33,867 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:33,867 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:33,867 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:33,867 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:33,868 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:33,868 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:33,868 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:48:33,868 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-05T15:48:33,868 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-05T15:48:33,869 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-05T15:48:33,869 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-05T15:48:33,869 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-05T15:48:33,870 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-05T15:48:33,870 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-05T15:48:33,870 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-05T15:48:33,871 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-05T15:48:33,871 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-05T15:48:33,871 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:33,871 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:33,872 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:33,872 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:33,872 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:33,873 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:33,873 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:33,914 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:48:33,937 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733413701337_ns2_test-17334136463821' hfile list 2024-12-05T15:48:33,939 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=13.2 K 2024-12-05T15:48:34,011 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local1803045381_0007_c5445776-a220-4b54-ade1-70d38d9daed7/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-05T15:48:34,012 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local1803045381_0007_c5445776-a220-4b54-ade1-70d38d9daed7/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-05T15:48:34,076 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-05T15:48:34,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741937_1113 (size=8413) 2024-12-05T15:48:34,510 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/1159a028de3844c4b30999226eb32483 output=hdfs://localhost:37233/backupUT/backup_1733413692606/ns2/test-17334136463821/archive/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/1159a028de3844c4b30999226eb32483 2024-12-05T15:48:34,510 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.019M/sec 2024-12-05T15:48:34,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741938_1114 (size=5134) 2024-12-05T15:48:34,938 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/d97c997a6e4d4d18b6b69960413bfcdd output=hdfs://localhost:37233/backupUT/backup_1733413692606/ns2/test-17334136463821/archive/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/d97c997a6e4d4d18b6b69960413bfcdd 2024-12-05T15:48:34,939 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5134 (5.0 K) time=0sec 0.012M/sec 2024-12-05T15:48:35,050 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-05T15:48:35,051 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-05T15:48:35,059 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733413701337_ns2_test-17334136463821 2024-12-05T15:48:35,059 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733413701337_ns2_test-17334136463821 finished. 2024-12-05T15:48:35,059 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733413692606_ns4:test-17334136463823 2024-12-05T15:48:35,060 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733413703498_ns4_test-17334136463823 to hdfs://localhost:37233/backupUT/backup_1733413692606/ns4/test-17334136463823/ 2024-12-05T15:48:35,060 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-05T15:48:35,084 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:37233, inputRoot=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:48:35,084 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_51721194_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37233/backupUT/backup_1733413692606/ns4/test-17334136463823, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37233/backupUT/backup_1733413692606/ns4/test-17334136463823/.hbase-snapshot/.tmp/snapshot_1733413703498_ns4_test-17334136463823 2024-12-05T15:48:35,086 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-05T15:48:35,090 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_1733413703498_ns4_test-17334136463823 to hdfs://localhost:37233/backupUT/backup_1733413692606/ns4/test-17334136463823/.hbase-snapshot/.tmp/snapshot_1733413703498_ns4_test-17334136463823 2024-12-05T15:48:35,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741940_1116 (size=443) 2024-12-05T15:48:35,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741939_1115 (size=86) 2024-12-05T15:48:35,503 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:35,504 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:35,504 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:36,274 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-14374100787735943036.jar 2024-12-05T15:48:36,274 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:36,274 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:36,275 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:36,275 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:36,275 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:36,275 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:36,275 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:36,275 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:36,276 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:36,276 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:48:36,276 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-05T15:48:36,276 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-05T15:48:36,276 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-05T15:48:36,277 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-05T15:48:36,277 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-05T15:48:36,277 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-05T15:48:36,277 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-05T15:48:36,277 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-05T15:48:36,277 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-05T15:48:36,278 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-05T15:48:36,278 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:36,278 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:36,278 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:36,278 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:36,279 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:36,279 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:36,279 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:36,310 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:48:36,333 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733413703498_ns4_test-17334136463823' hfile list 2024-12-05T15:48:36,412 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local1842274794_0008_a0097ba5-3ea4-45e2-b08a-e0bafa294788/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-05T15:48:36,412 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local1842274794_0008_a0097ba5-3ea4-45e2-b08a-e0bafa294788/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-05T15:48:37,447 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-05T15:48:37,449 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-05T15:48:37,459 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733413703498_ns4_test-17334136463823 2024-12-05T15:48:37,459 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733413703498_ns4_test-17334136463823 finished. 2024-12-05T15:48:37,460 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733413692606_test-1733413646382 2024-12-05T15:48:37,460 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733413705666_default_test-1733413646382 to hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/ 2024-12-05T15:48:37,460 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-05T15:48:37,484 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:37233, inputRoot=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:48:37,485 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_51721194_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/.hbase-snapshot/.tmp/snapshot_1733413705666_default_test-1733413646382 2024-12-05T15:48:37,486 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-05T15:48:37,492 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_1733413705666_default_test-1733413646382 to hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/.hbase-snapshot/.tmp/snapshot_1733413705666_default_test-1733413646382 2024-12-05T15:48:37,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741941_1117 (size=84) 2024-12-05T15:48:37,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741942_1118 (size=1347) 2024-12-05T15:48:37,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:37,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:37,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:38,673 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-521684269765426119.jar 2024-12-05T15:48:38,674 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:38,674 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:38,674 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:38,674 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:38,675 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:38,675 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:38,675 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:38,675 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:38,675 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:38,676 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:48:38,676 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-05T15:48:38,676 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-05T15:48:38,676 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-05T15:48:38,676 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-05T15:48:38,677 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-05T15:48:38,677 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-05T15:48:38,677 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-05T15:48:38,677 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-05T15:48:38,677 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-05T15:48:38,678 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-05T15:48:38,678 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:38,678 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:38,678 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:38,678 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:38,678 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-05T15:48:38,679 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:38,679 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:38,710 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:48:38,732 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733413705666_default_test-1733413646382' hfile list 2024-12-05T15:48:38,734 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=43.0 K 2024-12-05T15:48:38,807 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local1114035081_0009_ee80426e-764f-4b85-8131-45527aff3c3f/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-05T15:48:38,807 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local1114035081_0009_ee80426e-764f-4b85-8131-45527aff3c3f/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-05T15:48:38,868 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-05T15:48:38,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741943_1119 (size=13760) 2024-12-05T15:48:39,303 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/d940ad880da148ee8bac344a248bd8c3 output=hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/d940ad880da148ee8bac344a248bd8c3 2024-12-05T15:48:39,303 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=13760 (13.4 K) time=0sec 0.030M/sec 2024-12-05T15:48:39,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741944_1120 (size=10796) 2024-12-05T15:48:39,731 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/91dfff1e2dee4ba59bbdb059b1d0570f output=hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/91dfff1e2dee4ba59bbdb059b1d0570f 2024-12-05T15:48:39,732 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=10796 (10.5 K) time=0sec 0.024M/sec 2024-12-05T15:48:39,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741945_1121 (size=8413) 2024-12-05T15:48:40,166 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8 output=hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8 2024-12-05T15:48:40,166 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.019M/sec 2024-12-05T15:48:40,182 WARN [LocalJobRunner Map Task Executor #0 {}] mapred.Task(640): Task status: "copied 5.5 K/5.5 K (100.0%) from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/mobdir/data/default/test-1733413646382/d269731e9ae3c5ae370e1788a2e183c3/mob/d41d8cd98f00b204e9800998ecf8427e20241205070e4aa4b5934388ad16285a00c5ad73_24a988b9e3812c25fcebd4f4072c48f0 to hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/archive/data/default/test-1733413646382/d269731e9ae3c5ae370e1788a2e183c3/mob/d41d8cd98f00b204e9800998ecf8427e20241205070e4aa4b5934388ad16285a00c5ad73_24a988b9e3812c25fcebd4f4072c48f0" truncated to max limit (512 characters) 2024-12-05T15:48:40,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741946_1122 (size=5591) 2024-12-05T15:48:40,600 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/mobdir/data/default/test-1733413646382/d269731e9ae3c5ae370e1788a2e183c3/mob/d41d8cd98f00b204e9800998ecf8427e20241205070e4aa4b5934388ad16285a00c5ad73_24a988b9e3812c25fcebd4f4072c48f0 output=hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/archive/data/default/test-1733413646382/d269731e9ae3c5ae370e1788a2e183c3/mob/d41d8cd98f00b204e9800998ecf8427e20241205070e4aa4b5934388ad16285a00c5ad73_24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:48:40,600 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5591 (5.5 K) time=0sec 0.013M/sec 2024-12-05T15:48:40,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741947_1123 (size=5484) 2024-12-05T15:48:41,032 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/09842ffb0751416eaaeffe3bff7aefa1 output=hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/09842ffb0751416eaaeffe3bff7aefa1 2024-12-05T15:48:41,033 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-12-05T15:48:41,845 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-05T15:48:41,847 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-05T15:48:41,858 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733413705666_default_test-1733413646382 2024-12-05T15:48:41,859 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733413705666_default_test-1733413646382 finished. 2024-12-05T15:48:41,859 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): backup:system_bulk 2024-12-05T15:48:41,859 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns3:test-17334136463822 2024-12-05T15:48:41,859 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17334136463821 2024-12-05T15:48:41,859 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns4:test-17334136463823 2024-12-05T15:48:41,859 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1733413646382 2024-12-05T15:48:41,870 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733413692606 2024-12-05T15:48:41,870 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-12-05T15:48:41,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741948_1124 (size=176) 2024-12-05T15:48:42,276 INFO [Time-limited test {}] impl.BackupManifest(504): Manifest file stored to hdfs://localhost:37233/backupUT/backup_1733413692606/.backup.manifest 2024-12-05T15:48:42,276 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1733413692606 finished: type=FULL,tablelist=backup:system_bulk;ns3:test-17334136463822;ns2:test-17334136463821;ns4:test-17334136463823;test-1733413646382,targetRootDir=hdfs://localhost:37233/backupUT,startts=1733413696808,completets=1733413721870,bytescopied=0 2024-12-05T15:48:42,276 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-12-05T15:48:42,276 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733413697009_backup_system_bulk 2024-12-05T15:48:42,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733413697009_backup_system_bulk" type: DISABLED 2024-12-05T15:48:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733413697009_backup_system_bulk 2024-12-05T15:48:42,281 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733413697009_backup_system_bulk for backup backup_1733413692606 succeeded. 2024-12-05T15:48:42,281 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733413699177_ns3_test-17334136463822 2024-12-05T15:48:42,282 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733413699177_ns3_test-17334136463822" type: DISABLED 2024-12-05T15:48:42,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733413699177_ns3_test-17334136463822 2024-12-05T15:48:42,286 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733413699177_ns3_test-17334136463822 for backup backup_1733413692606 succeeded. 2024-12-05T15:48:42,286 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733413701337_ns2_test-17334136463821 2024-12-05T15:48:42,287 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733413701337_ns2_test-17334136463821" type: DISABLED 2024-12-05T15:48:42,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733413701337_ns2_test-17334136463821 2024-12-05T15:48:42,291 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733413701337_ns2_test-17334136463821 for backup backup_1733413692606 succeeded. 2024-12-05T15:48:42,291 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733413703498_ns4_test-17334136463823 2024-12-05T15:48:42,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733413703498_ns4_test-17334136463823" type: DISABLED 2024-12-05T15:48:42,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733413703498_ns4_test-17334136463823 2024-12-05T15:48:42,296 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733413703498_ns4_test-17334136463823 for backup backup_1733413692606 succeeded. 2024-12-05T15:48:42,296 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733413705666_default_test-1733413646382 2024-12-05T15:48:42,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733413705666_default_test-1733413646382" type: DISABLED 2024-12-05T15:48:42,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733413705666_default_test-1733413646382 2024-12-05T15:48:42,301 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733413705666_default_test-1733413646382 for backup backup_1733413692606 succeeded. 2024-12-05T15:48:42,302 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-05T15:48:42,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-12-05T15:48:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-05T15:48:42,313 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-05T15:48:42,316 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-05T15:48:42,321 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1733413692606 completed. 2024-12-05T15:48:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:45,266 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T15:48:47,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:47,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:47,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:47,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:47,401 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-05T15:48:47,406 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733413727346 2024-12-05T15:48:47,409 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733413692606} 2024-12-05T15:48:47,409 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-12-05T15:48:47,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:47,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:47,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:47,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:47,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:47,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:47,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:47,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:47,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:47,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:47,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-05T15:48:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733413727516 (current time:1733413727516). 2024-12-05T15:48:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-05T15:48:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-05T15:48:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bacc32d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:48:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:48:47,518 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:48:47,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:48:47,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:48:47,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4974cba8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:47,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:48:47,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:48:47,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:47,520 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:48:47,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49541f4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:48:47,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:48:47,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:48:47,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:48:47,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117. 2024-12-05T15:48:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:48:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:47,525 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:48:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-05T15:48:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-05T15:48:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-12-05T15:48:47,527 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-05T15:48:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T15:48:47,528 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-05T15:48:47,531 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-05T15:48:47,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741949_1125 (size=52) 2024-12-05T15:48:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T15:48:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T15:48:47,937 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-05T15:48:47,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af}] 2024-12-05T15:48:47,938 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af 2024-12-05T15:48:48,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34815 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2024-12-05T15:48:48,091 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:48:48,092 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing f422dcfec85c6b145613ea93a68881af 2/2 column families, dataSize=2.54 KB heapSize=4.22 KB 2024-12-05T15:48:48,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/meta/9503a1e16c26485f81461d2cc27faf9d is 158, key is trslm:hdfs://localhost:37233/backupUT\x00ns2:test-17334136463821/meta:log-roll-map/1733413721864/Put/seqid=0 2024-12-05T15:48:48,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741950_1126 (size=6646) 2024-12-05T15:48:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T15:48:48,509 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/meta/9503a1e16c26485f81461d2cc27faf9d 2024-12-05T15:48:48,526 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/session/28dfd966e0b946498f44d11bbb921e5d is 826, key is session:backup_1733413692606/session:context/1733413722315/Put/seqid=0 2024-12-05T15:48:48,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741951_1127 (size=5928) 2024-12-05T15:48:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T15:48:48,932 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/session/28dfd966e0b946498f44d11bbb921e5d 2024-12-05T15:48:48,944 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/meta/9503a1e16c26485f81461d2cc27faf9d as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/9503a1e16c26485f81461d2cc27faf9d 2024-12-05T15:48:48,950 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/9503a1e16c26485f81461d2cc27faf9d, entries=12, sequenceid=42, filesize=6.5 K 2024-12-05T15:48:48,952 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/session/28dfd966e0b946498f44d11bbb921e5d as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/28dfd966e0b946498f44d11bbb921e5d 2024-12-05T15:48:48,958 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/28dfd966e0b946498f44d11bbb921e5d, entries=2, sequenceid=42, filesize=5.8 K 2024-12-05T15:48:48,960 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~2.54 KB/2604, heapSize ~4.19 KB/4288, currentSize=0 B/0 for f422dcfec85c6b145613ea93a68881af in 868ms, sequenceid=42, compaction requested=true 2024-12-05T15:48:48,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for f422dcfec85c6b145613ea93a68881af: 2024-12-05T15:48:48,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. for snapshot_backup_system completed. 2024-12-05T15:48:48,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af.' region-info for snapshot=snapshot_backup_system 2024-12-05T15:48:48,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-05T15:48:48,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/e274103cce0e470c9f4fdd2b8874ffb0, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/f38659494b06472da7b8b4c06babbfa3, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/9503a1e16c26485f81461d2cc27faf9d] hfiles 2024-12-05T15:48:48,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/e274103cce0e470c9f4fdd2b8874ffb0 for snapshot=snapshot_backup_system 2024-12-05T15:48:48,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/f38659494b06472da7b8b4c06babbfa3 for snapshot=snapshot_backup_system 2024-12-05T15:48:48,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/9503a1e16c26485f81461d2cc27faf9d for snapshot=snapshot_backup_system 2024-12-05T15:48:48,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/ff33f5218d644a85986a494156767a9c, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/d9b315e36bf644cdb0a8feb6197ddaf0, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/03d3c4ccfd5d449f8c4a31f096312306, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/28dfd966e0b946498f44d11bbb921e5d] hfiles 2024-12-05T15:48:48,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/4): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/ff33f5218d644a85986a494156767a9c for snapshot=snapshot_backup_system 2024-12-05T15:48:48,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/4): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/d9b315e36bf644cdb0a8feb6197ddaf0 for snapshot=snapshot_backup_system 2024-12-05T15:48:48,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/4): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/03d3c4ccfd5d449f8c4a31f096312306 for snapshot=snapshot_backup_system 2024-12-05T15:48:48,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (4/4): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/28dfd966e0b946498f44d11bbb921e5d for snapshot=snapshot_backup_system 2024-12-05T15:48:48,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741952_1128 (size=330) 2024-12-05T15:48:49,369 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:48:49,369 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-05T15:48:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-05T15:48:49,371 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region f422dcfec85c6b145613ea93a68881af 2024-12-05T15:48:49,371 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af 2024-12-05T15:48:49,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-05T15:48:49,379 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-05T15:48:49,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SnapshotRegionProcedure f422dcfec85c6b145613ea93a68881af in 1.4370 sec 2024-12-05T15:48:49,380 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-05T15:48:49,381 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-05T15:48:49,381 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-05T15:48:49,383 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-05T15:48:49,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741953_1129 (size=967) 2024-12-05T15:48:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T15:48:49,801 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-05T15:48:49,812 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-05T15:48:49,813 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.hbase-snapshot/snapshot_backup_system 2024-12-05T15:48:49,814 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-05T15:48:49,815 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-12-05T15:48:49,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2900 sec 2024-12-05T15:48:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T15:48:51,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-05T15:48:51,687 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733413727346 started at 1733413731687. 2024-12-05T15:48:51,689 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733413727346 has been started. 2024-12-05T15:48:51,693 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [backup:system_bulk, ns2:test-17334136463821, ns3:test-17334136463822, test-1733413646382, ns4:test-17334136463823] 2024-12-05T15:48:51,697 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1733413683389for backupID backup_1733413727346 2024-12-05T15:48:51,697 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-12-05T15:48:51,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-12-05T15:48:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-05T15:48:51,698 INFO [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-05T15:48:51,698 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-05T15:48:51,698 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-05T15:48:51,698 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-05T15:48:51,699 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-05T15:48:51,699 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-05T15:48:51,814 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-05T15:48:51,815 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-05T15:48:51,815 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-05T15:48:51,815 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,815 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-05T15:48:51,815 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-05T15:48:51,816 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-05T15:48:51,816 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-05T15:48:51,816 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-05T15:48:51,816 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-05T15:48:51,817 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-05T15:48:51,817 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-05T15:48:51,817 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-05T15:48:51,817 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-05T15:48:51,817 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-05T15:48:51,817 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-05T15:48:51,817 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '5a29627a94df,34815,1733413637643' joining acquired barrier for procedure (rolllog) in zk 2024-12-05T15:48:51,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,866 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:51,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-05T15:48:51,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-05T15:48:51,867 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-05T15:48:51,867 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-05T15:48:51,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-05T15:48:51,868 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-05T15:48:51,868 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:51,869 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,869 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-05T15:48:51,869 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '5a29627a94df,34815,1733413637643' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-05T15:48:51,870 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@e177e0f[Count = 0] remaining members to acquire global barrier 2024-12-05T15:48:51,870 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-05T15:48:51,870 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:51,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:51,882 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:51,882 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:51,882 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-05T15:48:51,882 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,882 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-05T15:48:51,882 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-05T15:48:51,882 DEBUG [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,883 INFO [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733413696906 highest: 1733413696906 on 5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,883 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1:(num 1733413696884) roll requested 2024-12-05T15:48:51,898 DEBUG [regionserver/5a29627a94df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413731883, exclude list is [], retry=0 2024-12-05T15:48:51,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:48:51,902 INFO [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413696884 with entries=17, filesize=4.59 KB; new WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413731883 2024-12-05T15:48:51,902 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:48:51,902 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413696884 is not closed yet, will try archiving it next time 2024-12-05T15:48:51,902 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0:(num 1733413696906) roll requested 2024-12-05T15:48:51,902 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413683369 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413683369 2024-12-05T15:48:51,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741911_1087 (size=4710) 2024-12-05T15:48:51,913 DEBUG [regionserver/5a29627a94df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413731902, exclude list is [], retry=0 2024-12-05T15:48:51,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:48:51,917 INFO [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413696906 with entries=2, filesize=645 B; new WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413731902 2024-12-05T15:48:51,917 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:48:51,917 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413696906 is not closed yet, will try archiving it next time 2024-12-05T15:48:51,917 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5a29627a94df%2C34815%2C1733413637643.meta:.meta(num 1733413696926) roll requested 2024-12-05T15:48:51,917 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413664087 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413664087 2024-12-05T15:48:51,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741912_1088 (size=653) 2024-12-05T15:48:51,918 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413696906 not finished, retry = 0 2024-12-05T15:48:51,928 DEBUG [regionserver/5a29627a94df:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413731917.meta, exclude list is [], retry=0 2024-12-05T15:48:51,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43303,DS-b1b37f67-9005-4ca9-b0dc-6b20b3b6cfa6,DISK] 2024-12-05T15:48:51,932 INFO [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413696926.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413731917.meta 2024-12-05T15:48:51,932 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34553:34553)] 2024-12-05T15:48:51,932 DEBUG [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413696926.meta is not closed yet, will try archiving it next time 2024-12-05T15:48:51,934 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413696926.meta not finished, retry = 0 2024-12-05T15:48:51,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741913_1089 (size=93) 2024-12-05T15:48:51,943 DEBUG [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-12-05T15:48:51,943 INFO [rs(5a29627a94df,34815,1733413637643)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733413731902 on 5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:48:51,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:51,962 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-05T15:48:51,962 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '5a29627a94df,34815,1733413637643' in zk 2024-12-05T15:48:51,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,973 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-05T15:48:51,973 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-05T15:48:51,973 DEBUG [member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-05T15:48:51,973 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,973 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-05T15:48:51,973 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-05T15:48:51,974 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-05T15:48:51,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-05T15:48:51,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:51,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-05T15:48:51,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:51,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,978 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '5a29627a94df,34815,1733413637643': 2024-12-05T15:48:51,978 INFO [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-05T15:48:51,978 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '5a29627a94df,34815,1733413637643' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-05T15:48:51,978 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-05T15:48:51,978 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-05T15:48:51,978 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-05T15:48:51,979 INFO [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-05T15:48:51,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-05T15:48:51,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-05T15:48:51,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-05T15:48:51,989 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-05T15:48:51,989 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-05T15:48:51,989 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-05T15:48:51,989 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-05T15:48:51,989 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-05T15:48:51,989 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-05T15:48:51,989 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-05T15:48:51,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-05T15:48:51,990 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-05T15:48:51,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:51,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-05T15:48:51,992 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:51,992 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,992 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:48:51,993 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-05T15:48:51,994 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-05T15:48:51,994 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5a29627a94df,34815,1733413637643 2024-12-05T15:48:52,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:52,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-05T15:48:52,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-05T15:48:52,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-05T15:48:52,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-05T15:48:52,005 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-05T15:48:52,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5a29627a94df,34815,1733413637643 2024-12-05T15:48:52,005 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-05T15:48:52,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:52,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-05T15:48:52,005 DEBUG [(5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-05T15:48:52,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-05T15:48:52,005 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-05T15:48:52,005 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-05T15:48:52,005 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2024-12-05T15:48:52,006 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-05T15:48:52,006 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-05T15:48:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-05T15:48:52,020 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413696906 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413696906 2024-12-05T15:48:52,021 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {5a29627a94df:34815=1733413683389} newestTimestamps: {5a29627a94df:34815=1733413696906} 2024-12-05T15:48:52,024 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 5a29627a94df%2C34815%2C1733413637643.meta.1733413641984.meta 2024-12-05T15:48:52,024 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 5a29627a94df%2C34815%2C1733413637643.meta.1733413683405.meta 2024-12-05T15:48:52,024 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 5a29627a94df%2C34815%2C1733413637643.meta.1733413696926.meta 2024-12-05T15:48:52,024 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 5a29627a94df%2C34815%2C1733413637643.meta.1733413731917.meta 2024-12-05T15:48:52,024 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413731902 2024-12-05T15:48:52,024 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413696884 2024-12-05T15:48:52,024 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413731883 2024-12-05T15:48:52,025 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs/5a29627a94df%2C34815%2C1733413637643.meta.1733413664106.meta 2024-12-05T15:48:52,035 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413696926.meta to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs/5a29627a94df%2C34815%2C1733413637643.meta.1733413696926.meta 2024-12-05T15:48:52,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741957_1133 (size=640) 2024-12-05T15:48:52,440 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:backup:system_bulk target: hdfs://localhost:37233/backupUT/backup_1733413727346/backup/system_bulk descriptor: 'backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:48:52,440 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-05T15:48:52,445 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table backup:system_bulk 2024-12-05T15:48:52,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741958_1134 (size=45) 2024-12-05T15:48:52,854 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table backup:system_bulk 2024-12-05T15:48:52,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741959_1135 (size=390) 2024-12-05T15:48:53,273 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns3:test-17334136463822 target: hdfs://localhost:37233/backupUT/backup_1733413727346/ns3/test-17334136463822 descriptor: 'ns3:test-17334136463822', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:48:53,273 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-05T15:48:53,277 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns3:test-17334136463822 2024-12-05T15:48:53,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741960_1136 (size=50) 2024-12-05T15:48:53,687 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns3:test-17334136463822 2024-12-05T15:48:53,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741961_1137 (size=364) 2024-12-05T15:48:54,102 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17334136463821 target: hdfs://localhost:37233/backupUT/backup_1733413727346/ns2/test-17334136463821 descriptor: 'ns2:test-17334136463821', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:48:54,102 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-05T15:48:54,107 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17334136463821 2024-12-05T15:48:54,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741962_1138 (size=50) 2024-12-05T15:48:54,114 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17334136463821 2024-12-05T15:48:54,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741963_1139 (size=390) 2024-12-05T15:48:54,529 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns4:test-17334136463823 target: hdfs://localhost:37233/backupUT/backup_1733413727346/ns4/test-17334136463823 descriptor: 'ns4:test-17334136463823', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:48:54,529 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-05T15:48:54,533 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns4:test-17334136463823 2024-12-05T15:48:54,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741964_1140 (size=50) 2024-12-05T15:48:54,942 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns4:test-17334136463823 2024-12-05T15:48:54,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741965_1141 (size=950) 2024-12-05T15:48:55,362 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1733413646382 target: hdfs://localhost:37233/backupUT/backup_1733413727346/default/test-1733413646382 descriptor: 'test-1733413646382', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:48:55,362 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-05T15:48:55,368 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1733413646382 2024-12-05T15:48:55,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741966_1142 (size=53) 2024-12-05T15:48:55,775 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1733413646382 2024-12-05T15:48:55,792 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:37233/backupUT/.tmp/backup_1733413727346 from hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413696884;hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413696906 2024-12-05T15:48:55,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@250572a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:55,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:48:55,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:48:55,794 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:48:55,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:48:55,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:48:55,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ec1ba3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:55,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:48:55,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:48:55,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:55,795 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:48:55,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@686b7014, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:55,796 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:48:55,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:48:55,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@e292787 2024-12-05T15:48:55,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:48:55,797 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:48:55,801 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-05T15:48:55,801 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table backup:system_bulk 2024-12-05T15:48:55,801 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:48:55,801 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:48:55,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:48:55,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:48:55,803 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:48:55,804 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for backup:system_bulk: backup:system_bulk; 2024-12-05T15:48:55,804 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17334136463821 2024-12-05T15:48:55,805 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17334136463821,, stopping at row=ns2:test-17334136463821 ,, for max=2147483647 with caching=100 2024-12-05T15:48:55,806 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17334136463821: ns2:test-17334136463821; 2024-12-05T15:48:55,806 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns3:test-17334136463822 2024-12-05T15:48:55,806 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns3:test-17334136463822,, stopping at row=ns3:test-17334136463822 ,, for max=2147483647 with caching=100 2024-12-05T15:48:55,808 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns3:test-17334136463822: ns3:test-17334136463822; 2024-12-05T15:48:55,808 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1733413646382 2024-12-05T15:48:55,808 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733413646382,, stopping at row=test-1733413646382 ,, for max=2147483647 with caching=100 2024-12-05T15:48:55,810 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1733413646382: test-1733413646382; 2024-12-05T15:48:55,810 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns4:test-17334136463823 2024-12-05T15:48:55,810 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns4:test-17334136463823,, stopping at row=ns4:test-17334136463823 ,, for max=2147483647 with caching=100 2024-12-05T15:48:55,812 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns4:test-17334136463823: ns4:test-17334136463823; 2024-12-05T15:48:55,812 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 5 reduce partitions to match current region count for all tables 2024-12-05T15:48:55,812 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/partitions_d16dba8a-7c7a-4b2b-8765-00cc5c622663 2024-12-05T15:48:55,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741967_1143 (size=324) 2024-12-05T15:48:56,223 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,224 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,225 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,984 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-7523155262993831039.jar 2024-12-05T15:48:56,984 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,985 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,985 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,985 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,985 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,985 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,985 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,986 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,986 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,986 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:48:56,986 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-05T15:48:56,986 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-05T15:48:56,987 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-05T15:48:56,987 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-05T15:48:56,987 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-05T15:48:56,987 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-05T15:48:56,987 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-05T15:48:56,987 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-05T15:48:56,988 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-05T15:48:56,988 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-05T15:48:56,988 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,988 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,988 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,988 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,989 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,989 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:48:56,989 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:48:56,989 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: backup:system_bulk,ns2:test-17334136463821,ns3:test-17334136463822,default:test-1733413646382,ns4:test-17334136463823 2024-12-05T15:48:56,989 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:48:56,989 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:188) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:48:56,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:56,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:56,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:48:56,990 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:48:57,430 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 24a988b9e3812c25fcebd4f4072c48f0, had cached 0 bytes from a total of 38453 2024-12-05T15:48:57,749 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-9891960291105567931.jar 2024-12-05T15:48:57,759 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:48:57,781 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413696884 for WAL files 2024-12-05T15:48:57,782 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413696884 2024-12-05T15:48:57,782 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413696906 for WAL files 2024-12-05T15:48:57,782 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413696906 2024-12-05T15:48:57,853 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local1496225523_0010_08ad242e-a9ef-4768-87e1-f39c683574ea/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-05T15:48:57,854 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local1496225523_0010_08ad242e-a9ef-4768-87e1-f39c683574ea/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-05T15:48:57,899 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413696884 for hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1.1733413696884 (-9223372036854775808:9223372036854775807) length:4710 2024-12-05T15:48:57,903 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-12-05T15:48:57,903 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-12-05T15:48:57,936 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413696906 for hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413696906 (-9223372036854775808:9223372036854775807) length:653 2024-12-05T15:48:57,938 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-12-05T15:48:57,938 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-12-05T15:48:58,021 DEBUG [pool-260-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4492e64f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:58,021 DEBUG [pool-260-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:48:58,022 DEBUG [pool-260-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:48:58,023 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:48:58,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:48:58,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:48:58,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b5914c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:58,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:48:58,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:48:58,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:58,024 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:48:58,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1930118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:48:58,025 DEBUG [pool-260-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:48:58,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:48:58,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:48:58,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:48:58,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733413646382', row='row-test-1733413646382-2-0', locateType=CURRENT is [region=test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0., hostname=5a29627a94df,34815,1733413637643, seqNum=338] 2024-12-05T15:48:58,029 INFO [pool-260-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-260-thread-1. 2024-12-05T15:48:58,029 DEBUG [pool-260-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:48:58,029 DEBUG [pool-260-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:58,029 DEBUG [pool-260-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:48:58,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:48:58,030 DEBUG [pool-260-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733413646382-2-0] 2024-12-05T15:48:58,030 DEBUG [pool-260-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5a29627a94df 2024-12-05T15:48:58,033 DEBUG [pool-260-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/backupUT/.tmp/backup_1733413727346/_temporary/0/_temporary/attempt_local1496225523_0010_r_000004_0/default/test-1733413646382/f2/f294e4172be34af5990ccf7377ca6be0 is 58, key is row-test-1733413646382-2-0/f2:q1/1733413722337/Put/seqid=0 2024-12-05T15:48:58,035 WARN [Thread-1516 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5a29627a94df:34815] Specified favored nodes: [5a29627a94df:34815] 2024-12-05T15:48:58,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741968_1144 (size=5367) 2024-12-05T15:48:58,898 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:37233/backupUT 2024-12-05T15:48:58,899 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1733413727346 2024-12-05T15:48:58,899 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-12-05T15:48:58,909 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:37233/backupUT/.tmp/backup_1733413727346, hdfs://localhost:37233/backupUT] 2024-12-05T15:48:58,957 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:48:59,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741969_1145 (size=5367) 2024-12-05T15:48:59,554 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-12-05T15:48:59,557 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1733413727346 2024-12-05T15:48:59,557 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 5367 bytes copied." 2024-12-05T15:48:59,557 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local419336262_0011 completed: true true 2024-12-05T15:48:59,558 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=6688404 FILE: Number of bytes written=17225195 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=410793 HDFS: Number of bytes written=2548188 HDFS: Number of read operations=1171 HDFS: Number of large read operations=0 HDFS: Number of write operations=421 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=1482 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=5367 Bytes Copied=5367 Bytes Expected=5367 Files Copied=2 DIR_COPY=4 2024-12-05T15:48:59,558 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:37233/backupUT/.tmp/backup_1733413727346 to hdfs://localhost:37233/backupUT finished. 2024-12-05T15:48:59,576 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733413727346 2024-12-05T15:48:59,580 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733413692606} 2024-12-05T15:48:59,580 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-12-05T15:48:59,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741970_1146 (size=648) 2024-12-05T15:48:59,986 INFO [Time-limited test {}] impl.BackupManifest(504): Manifest file stored to hdfs://localhost:37233/backupUT/backup_1733413727346/.backup.manifest 2024-12-05T15:48:59,987 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1733413727346 finished: type=INCREMENTAL,tablelist=backup:system_bulk;ns3:test-17334136463822;ns2:test-17334136463821;ns4:test-17334136463823;test-1733413646382,targetRootDir=hdfs://localhost:37233/backupUT,startts=1733413731687,completets=1733413739576,bytescopied=0 2024-12-05T15:48:59,987 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-05T15:48:59,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-12-05T15:49:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-05T15:49:00,002 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-05T15:49:00,006 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-05T15:49:00,012 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1733413727346 completed. 2024-12-05T15:49:00,012 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-12-05T15:49:00,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:00,027 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:37233/backupUT/backup_1733413727346/.backup.manifest 2024-12-05T15:49:00,030 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733413727346/.backup.manifest 2024-12-05T15:49:00,030 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(196): Restoring full backup_1733413657238 2024-12-05T15:49:00,033 INFO [Time-limited test {}] impl.RestoreTablesClient(100): HBase table table1 does not exist. It will be created during restore process 2024-12-05T15:49:00,034 INFO [Time-limited test {}] impl.RestoreTablesClient(100): HBase table ns2:table2 does not exist. It will be created during restore process 2024-12-05T15:49:00,035 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:37233/backupUT/backup_1733413657238/.backup.manifest 2024-12-05T15:49:00,037 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733413657238/.backup.manifest 2024-12-05T15:49:00,040 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:37233/backupUT/backup_1733413657238/.backup.manifest 2024-12-05T15:49:00,042 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733413657238/.backup.manifest 2024-12-05T15:49:00,042 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'test-1733413646382' to 'table1' from full backup image hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382 2024-12-05T15:49:00,050 INFO [Time-limited test {}] util.RestoreTool(491): Creating target table 'table1' 2024-12-05T15:49:00,050 DEBUG [Time-limited test {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:49:00,051 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f in region [hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0] 2024-12-05T15:49:00,055 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8 first=row0 last=row98 2024-12-05T15:49:00,055 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f3 in region [hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0] 2024-12-05T15:49:00,058 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f3/c7bc8ee8395f42e39c50f2005338ee85 first=row-test-1733413646382-3-0 last=row-test-1733413646382-3-5 2024-12-05T15:49:00,058 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob in region [hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0] 2024-12-05T15:49:00,061 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/09842ffb0751416eaaeffe3bff7aefa1 first=row-test-1733413646382-3-0 last=row-test-1733413646382-3-5 2024-12-05T15:49:00,066 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'table1', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:49:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-12-05T15:49:00,068 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T15:49:00,068 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:00,068 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 62 2024-12-05T15:49:00,069 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T15:49:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T15:49:00,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741971_1147 (size=938) 2024-12-05T15:49:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T15:49:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T15:49:00,429 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region b7877185d3b50523892995990a3cd728, had cached 0 bytes from a total of 13547 2024-12-05T15:49:00,479 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 98789158a6f27fe53f9525b6fc318ac3, NAME => 'table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3.', STARTKEY => '', ENDKEY => 'row0'}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:49:00,480 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8449bae89aa1e699ba056d7b9b3032d3, NAME => 'table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3.', STARTKEY => 'row0', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:49:00,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741972_1148 (size=45) 2024-12-05T15:49:00,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741973_1149 (size=45) 2024-12-05T15:49:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T15:49:00,892 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(898): Instantiated table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:49:00,892 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:49:00,893 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1722): Closing 8449bae89aa1e699ba056d7b9b3032d3, disabling compactions & flushes 2024-12-05T15:49:00,893 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing 98789158a6f27fe53f9525b6fc318ac3, disabling compactions & flushes 2024-12-05T15:49:00,893 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. 2024-12-05T15:49:00,893 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1755): Closing region table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. 2024-12-05T15:49:00,893 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. 2024-12-05T15:49:00,893 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. 2024-12-05T15:49:00,893 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. after waiting 0 ms 2024-12-05T15:49:00,893 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. after waiting 0 ms 2024-12-05T15:49:00,893 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. 2024-12-05T15:49:00,893 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. 2024-12-05T15:49:00,894 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1973): Closed table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. 2024-12-05T15:49:00,894 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. 2024-12-05T15:49:00,894 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for 98789158a6f27fe53f9525b6fc318ac3: Waiting for close lock at 1733413740892Disabling compacts and flushes for region at 1733413740892Disabling writes for close at 1733413740893 (+1 ms)Writing region close event to WAL at 1733413740893Closed at 1733413740893 2024-12-05T15:49:00,894 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8449bae89aa1e699ba056d7b9b3032d3: Waiting for close lock at 1733413740892Disabling compacts and flushes for region at 1733413740892Disabling writes for close at 1733413740893 (+1 ms)Writing region close event to WAL at 1733413740893Closed at 1733413740893 2024-12-05T15:49:00,897 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T15:49:00,898 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733413740897"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733413740897"}]},"ts":"1733413740897"} 2024-12-05T15:49:00,899 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733413740897"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733413740897"}]},"ts":"1733413740897"} 2024-12-05T15:49:00,904 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 2 regions to meta. 2024-12-05T15:49:00,905 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T15:49:00,906 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413740905"}]},"ts":"1733413740905"} 2024-12-05T15:49:00,907 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-12-05T15:49:00,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=98789158a6f27fe53f9525b6fc318ac3, ASSIGN}, {pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=8449bae89aa1e699ba056d7b9b3032d3, ASSIGN}] 2024-12-05T15:49:00,909 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=8449bae89aa1e699ba056d7b9b3032d3, ASSIGN 2024-12-05T15:49:00,909 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=98789158a6f27fe53f9525b6fc318ac3, ASSIGN 2024-12-05T15:49:00,909 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=8449bae89aa1e699ba056d7b9b3032d3, ASSIGN; state=OFFLINE, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=false 2024-12-05T15:49:00,909 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=98789158a6f27fe53f9525b6fc318ac3, ASSIGN; state=OFFLINE, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=false 2024-12-05T15:49:01,060 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=98789158a6f27fe53f9525b6fc318ac3, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:01,060 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=8449bae89aa1e699ba056d7b9b3032d3, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:01,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=98789158a6f27fe53f9525b6fc318ac3, ASSIGN because future has completed 2024-12-05T15:49:01,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 98789158a6f27fe53f9525b6fc318ac3, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:49:01,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=8449bae89aa1e699ba056d7b9b3032d3, ASSIGN because future has completed 2024-12-05T15:49:01,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8449bae89aa1e699ba056d7b9b3032d3, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:49:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T15:49:01,225 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] handler.AssignRegionHandler(132): Open table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. 2024-12-05T15:49:01,226 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(7752): Opening region: {ENCODED => 8449bae89aa1e699ba056d7b9b3032d3, NAME => 'table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3.', STARTKEY => 'row0', ENDKEY => ''} 2024-12-05T15:49:01,227 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:49:01,227 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:01,227 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(898): Instantiated table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:49:01,227 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(7794): checking encryption for 8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:01,227 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(7797): checking classloading for 8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:01,230 INFO [StoreOpener-8449bae89aa1e699ba056d7b9b3032d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:01,232 INFO [StoreOpener-8449bae89aa1e699ba056d7b9b3032d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8449bae89aa1e699ba056d7b9b3032d3 columnFamilyName f 2024-12-05T15:49:01,232 DEBUG [StoreOpener-8449bae89aa1e699ba056d7b9b3032d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:01,233 INFO [StoreOpener-8449bae89aa1e699ba056d7b9b3032d3-1 {}] regionserver.HStore(327): Store=8449bae89aa1e699ba056d7b9b3032d3/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:01,233 INFO [StoreOpener-8449bae89aa1e699ba056d7b9b3032d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:01,235 INFO [StoreOpener-8449bae89aa1e699ba056d7b9b3032d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8449bae89aa1e699ba056d7b9b3032d3 columnFamilyName f3 2024-12-05T15:49:01,235 DEBUG [StoreOpener-8449bae89aa1e699ba056d7b9b3032d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:01,236 INFO [StoreOpener-8449bae89aa1e699ba056d7b9b3032d3-1 {}] regionserver.HStore(327): Store=8449bae89aa1e699ba056d7b9b3032d3/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:01,236 INFO [StoreOpener-8449bae89aa1e699ba056d7b9b3032d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:01,238 INFO [StoreOpener-8449bae89aa1e699ba056d7b9b3032d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8449bae89aa1e699ba056d7b9b3032d3 columnFamilyName mob 2024-12-05T15:49:01,240 DEBUG [StoreOpener-8449bae89aa1e699ba056d7b9b3032d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:01,241 INFO [StoreOpener-8449bae89aa1e699ba056d7b9b3032d3-1 {}] regionserver.HStore(327): Store=8449bae89aa1e699ba056d7b9b3032d3/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:01,241 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1038): replaying wal for 8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:01,242 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:01,242 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:01,243 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1048): stopping wal replay for 8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:01,243 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1060): Cleaning up temporary data for 8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:01,244 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T15:49:01,245 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1093): writing seq id for 8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:01,248 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T15:49:01,248 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1114): Opened 8449bae89aa1e699ba056d7b9b3032d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11258634720, jitterRate=0.04854206740856171}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T15:49:01,248 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:01,249 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1006): Region open journal for 8449bae89aa1e699ba056d7b9b3032d3: Running coprocessor pre-open hook at 1733413741227Writing region info on filesystem at 1733413741227Initializing all the Stores at 1733413741229 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413741229Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413741229Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413741229Cleaning up temporary data from old regions at 1733413741243 (+14 ms)Running coprocessor post-open hooks at 1733413741248 (+5 ms)Region opened successfully at 1733413741249 (+1 ms) 2024-12-05T15:49:01,250 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3., pid=66, masterSystemTime=1733413741216 2024-12-05T15:49:01,252 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. 2024-12-05T15:49:01,252 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] handler.AssignRegionHandler(153): Opened table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. 2024-12-05T15:49:01,252 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. 2024-12-05T15:49:01,252 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 98789158a6f27fe53f9525b6fc318ac3, NAME => 'table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3.', STARTKEY => '', ENDKEY => 'row0'} 2024-12-05T15:49:01,253 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=8449bae89aa1e699ba056d7b9b3032d3, regionState=OPEN, openSeqNum=2, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:01,253 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:49:01,253 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:01,253 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:49:01,253 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:01,253 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:01,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8449bae89aa1e699ba056d7b9b3032d3, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:49:01,255 INFO [StoreOpener-98789158a6f27fe53f9525b6fc318ac3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:01,257 INFO [StoreOpener-98789158a6f27fe53f9525b6fc318ac3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98789158a6f27fe53f9525b6fc318ac3 columnFamilyName f 2024-12-05T15:49:01,257 DEBUG [StoreOpener-98789158a6f27fe53f9525b6fc318ac3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:01,257 INFO [StoreOpener-98789158a6f27fe53f9525b6fc318ac3-1 {}] regionserver.HStore(327): Store=98789158a6f27fe53f9525b6fc318ac3/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:01,258 INFO [StoreOpener-98789158a6f27fe53f9525b6fc318ac3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:01,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=64 2024-12-05T15:49:01,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 8449bae89aa1e699ba056d7b9b3032d3, server=5a29627a94df,34815,1733413637643 in 191 msec 2024-12-05T15:49:01,259 INFO [StoreOpener-98789158a6f27fe53f9525b6fc318ac3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98789158a6f27fe53f9525b6fc318ac3 columnFamilyName f3 2024-12-05T15:49:01,259 DEBUG [StoreOpener-98789158a6f27fe53f9525b6fc318ac3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:01,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=62, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=8449bae89aa1e699ba056d7b9b3032d3, ASSIGN in 350 msec 2024-12-05T15:49:01,260 INFO [StoreOpener-98789158a6f27fe53f9525b6fc318ac3-1 {}] regionserver.HStore(327): Store=98789158a6f27fe53f9525b6fc318ac3/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:01,260 INFO [StoreOpener-98789158a6f27fe53f9525b6fc318ac3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:01,262 INFO [StoreOpener-98789158a6f27fe53f9525b6fc318ac3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98789158a6f27fe53f9525b6fc318ac3 columnFamilyName mob 2024-12-05T15:49:01,262 DEBUG [StoreOpener-98789158a6f27fe53f9525b6fc318ac3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:01,263 INFO [StoreOpener-98789158a6f27fe53f9525b6fc318ac3-1 {}] regionserver.HStore(327): Store=98789158a6f27fe53f9525b6fc318ac3/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:01,263 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:01,264 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:01,264 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:01,265 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:01,265 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:01,265 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T15:49:01,266 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:01,268 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T15:49:01,269 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 98789158a6f27fe53f9525b6fc318ac3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10618204000, jitterRate=-0.011102691292762756}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T15:49:01,269 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:01,269 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 98789158a6f27fe53f9525b6fc318ac3: Running coprocessor pre-open hook at 1733413741253Writing region info on filesystem at 1733413741253Initializing all the Stores at 1733413741254 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413741254Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413741255 (+1 ms)Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413741255Cleaning up temporary data from old regions at 1733413741265 (+10 ms)Running coprocessor post-open hooks at 1733413741269 (+4 ms)Region opened successfully at 1733413741269 2024-12-05T15:49:01,269 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3., pid=65, masterSystemTime=1733413741216 2024-12-05T15:49:01,271 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. 2024-12-05T15:49:01,271 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. 2024-12-05T15:49:01,272 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=98789158a6f27fe53f9525b6fc318ac3, regionState=OPEN, openSeqNum=2, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:01,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 98789158a6f27fe53f9525b6fc318ac3, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:49:01,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=63 2024-12-05T15:49:01,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 98789158a6f27fe53f9525b6fc318ac3, server=5a29627a94df,34815,1733413637643 in 211 msec 2024-12-05T15:49:01,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-05T15:49:01,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=98789158a6f27fe53f9525b6fc318ac3, ASSIGN in 368 msec 2024-12-05T15:49:01,278 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T15:49:01,278 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413741278"}]},"ts":"1733413741278"} 2024-12-05T15:49:01,280 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-12-05T15:49:01,280 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T15:49:01,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 1.2140 sec 2024-12-05T15:49:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T15:49:02,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-12-05T15:49:02,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-05T15:49:02,233 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0 from hbase tables test-1733413646382 to tables table1 2024-12-05T15:49:02,233 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore test-1733413646382 into table1 2024-12-05T15:49:02,239 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413742233 from hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:49:02,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26db0f63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:02,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:02,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:02,242 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:02,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:02,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:02,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7378e6ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:02,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:02,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:02,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:02,244 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:02,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ed9a0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:02,245 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:02,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:02,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7bba23d5 2024-12-05T15:49:02,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:02,246 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:02,247 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-05T15:49:02,247 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table table1 2024-12-05T15:49:02,247 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-05T15:49:02,247 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:02,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:02,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:02,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:02,251 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: 2024-12-05T15:49:02,251 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: row0 2024-12-05T15:49:02,251 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-12-05T15:49:02,251 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/partitions_5ee269a3-27ac-404c-99bc-382681bc0c10 2024-12-05T15:49:02,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741974_1150 (size=177) 2024-12-05T15:49:02,660 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:02,661 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:02,661 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:02,934 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-12-05T15:49:03,420 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-3565058294532087156.jar 2024-12-05T15:49:03,420 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:03,420 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:03,421 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:03,421 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:03,421 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:03,421 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:03,422 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:03,422 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:03,422 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:03,422 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:49:03,422 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-05T15:49:03,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-05T15:49:03,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-05T15:49:03,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-05T15:49:03,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-05T15:49:03,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-05T15:49:03,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-05T15:49:03,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-05T15:49:03,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-05T15:49:03,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-05T15:49:03,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:03,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:03,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:03,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:03,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:03,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:03,427 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:49:03,427 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: table1 2024-12-05T15:49:03,427 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:49:03,427 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:197) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:03,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:03,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:03,427 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:03,427 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-12-05T15:49:03,428 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:49:03,438 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:49:03,449 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-12-05T15:49:03,531 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local1447355149_0012_83f6bb43-8c4b-4689-b142-07b8d2a3047d/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-05T15:49:03,531 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local1447355149_0012_83f6bb43-8c4b-4689-b142-07b8d2a3047d/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-05T15:49:03,579 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8 2024-12-05T15:49:03,582 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-05T15:49:03,619 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/09842ffb0751416eaaeffe3bff7aefa1 2024-12-05T15:49:03,622 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-05T15:49:03,642 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f3/c7bc8ee8395f42e39c50f2005338ee85 2024-12-05T15:49:03,645 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-05T15:49:03,670 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f462048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:03,670 DEBUG [pool-267-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:03,671 DEBUG [pool-267-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:03,671 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:03,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:03,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:03,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b82d3d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:03,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:03,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:03,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:03,673 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:03,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@170713b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:03,674 DEBUG [pool-267-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:03,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:03,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:03,676 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:03,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1733413646382-3-0', locateType=CURRENT is [region=table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:03,678 INFO [pool-267-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-267-thread-1. 2024-12-05T15:49:03,678 DEBUG [pool-267-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:03,678 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:03,678 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:03,678 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733413646382-3-0] 2024-12-05T15:49:03,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:03,678 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5a29627a94df 2024-12-05T15:49:03,683 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e5eed60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:03,683 DEBUG [pool-267-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:03,683 DEBUG [pool-267-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:03,684 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:03,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:03,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:03,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be023a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:03,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:03,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:03,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:03,686 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:03,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cf30767, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:03,687 DEBUG [pool-267-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:03,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:03,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:03,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:03,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1733413646382-3-0', locateType=CURRENT is [region=table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:03,691 INFO [pool-267-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-267-thread-1. 2024-12-05T15:49:03,691 DEBUG [pool-267-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:03,691 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:03,691 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:03,691 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:03,691 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733413646382-3-0] 2024-12-05T15:49:03,691 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5a29627a94df 2024-12-05T15:49:03,696 DEBUG [pool-267-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413742233/_temporary/0/_temporary/attempt_local1447355149_0012_r_000000_0/f3/178e4c55e67d40638256035b48c2f321 is 58, key is row-test-1733413646382-3-0/f3:q1/1733413657204/Put/seqid=0 2024-12-05T15:49:03,697 WARN [Thread-1635 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5a29627a94df:34815] Specified favored nodes: [5a29627a94df:34815] 2024-12-05T15:49:03,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741975_1151 (size=5530) 2024-12-05T15:49:04,104 DEBUG [pool-267-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413742233/_temporary/0/_temporary/attempt_local1447355149_0012_r_000000_0/mob/744bacfadce540928e56cdcf38de555b is 59, key is row-test-1733413646382-3-0/mob:q1/1733413657221/Put/seqid=0 2024-12-05T15:49:04,107 WARN [Thread-1637 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5a29627a94df:34815] Specified favored nodes: [5a29627a94df:34815] 2024-12-05T15:49:04,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741976_1152 (size=5539) 2024-12-05T15:49:04,544 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@430e06b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:04,545 DEBUG [pool-267-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:04,545 DEBUG [pool-267-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:04,545 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:04,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:04,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:04,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef0ddb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:04,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:04,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:04,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:04,547 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:04,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74e9d3b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:04,548 DEBUG [pool-267-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:04,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:04,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:04,550 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:04,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:04,552 INFO [pool-267-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-267-thread-1. 2024-12-05T15:49:04,552 DEBUG [pool-267-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:04,552 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:04,552 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:04,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:04,552 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row0] 2024-12-05T15:49:04,552 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5a29627a94df 2024-12-05T15:49:04,563 DEBUG [pool-267-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413742233/_temporary/0/_temporary/attempt_local1447355149_0012_r_000001_0/f/08aeeeb8bb124367bf6ff13ad871d915 is 37, key is row10/f:q1/1733413649096/Put/seqid=0 2024-12-05T15:49:04,564 WARN [Thread-1652 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5a29627a94df:34815] Specified favored nodes: [5a29627a94df:34815] 2024-12-05T15:49:04,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741977_1153 (size=8742) 2024-12-05T15:49:05,574 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413742233 2024-12-05T15:49:05,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4040e2c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,575 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:05,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:05,577 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:05,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:05,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:05,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a6c927b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:05,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:05,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,579 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:05,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5108f659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,580 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:05,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:05,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:05,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:05,585 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413742233/_SUCCESS 2024-12-05T15:49:05,593 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:05,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:05,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@360d968e 2024-12-05T15:49:05,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:05,595 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:05,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:05,606 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-05T15:49:05,616 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413742233/f/08aeeeb8bb124367bf6ff13ad871d915 first=Optional[row0] last=Optional[row98] 2024-12-05T15:49:05,619 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413742233/f3/178e4c55e67d40638256035b48c2f321 first=Optional[row-test-1733413646382-3-0] last=Optional[row-test-1733413646382-3-5] 2024-12-05T15:49:05,622 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413742233/mob/744bacfadce540928e56cdcf38de555b first=Optional[row-test-1733413646382-3-0] last=Optional[row-test-1733413646382-3-5] 2024-12-05T15:49:05,639 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413742233/f3/178e4c55e67d40638256035b48c2f321 for inclusion in 98789158a6f27fe53f9525b6fc318ac3/f3 2024-12-05T15:49:05,639 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413742233/f/08aeeeb8bb124367bf6ff13ad871d915 for inclusion in 8449bae89aa1e699ba056d7b9b3032d3/f 2024-12-05T15:49:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HStore(626): HFile bounds: first=row0 last=row98 2024-12-05T15:49:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(626): HFile bounds: first=row-test-1733413646382-3-0 last=row-test-1733413646382-3-5 2024-12-05T15:49:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-12-05T15:49:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HStore(628): Region bounds: first=row0 last= 2024-12-05T15:49:05,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413742233/mob/744bacfadce540928e56cdcf38de555b for inclusion in 98789158a6f27fe53f9525b6fc318ac3/mob 2024-12-05T15:49:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HRegion(2603): Flush status journal for 8449bae89aa1e699ba056d7b9b3032d3: 2024-12-05T15:49:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413742233/f/08aeeeb8bb124367bf6ff13ad871d915 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__rcp8sjpkrh8eahn8u6adqrh4bid20d7ggar9o7uh62pprq5e4kp15me9dkqqjcc3/f/08aeeeb8bb124367bf6ff13ad871d915 2024-12-05T15:49:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(626): HFile bounds: first=row-test-1733413646382-3-0 last=row-test-1733413646382-3-5 2024-12-05T15:49:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-12-05T15:49:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__rcp8sjpkrh8eahn8u6adqrh4bid20d7ggar9o7uh62pprq5e4kp15me9dkqqjcc3/f/08aeeeb8bb124367bf6ff13ad871d915 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3/f/1433265d50e14b3dba3552818319cd14_SeqId_4_ 2024-12-05T15:49:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HRegion(2603): Flush status journal for 98789158a6f27fe53f9525b6fc318ac3: 2024-12-05T15:49:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41bbd4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413742233/f3/178e4c55e67d40638256035b48c2f321 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__rcp8sjpkrh8eahn8u6adqrh4bid20d7ggar9o7uh62pprq5e4kp15me9dkqqjcc3/f3/178e4c55e67d40638256035b48c2f321 2024-12-05T15:49:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:05,652 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:05,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:05,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:05,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3577e367, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:05,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:05,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__rcp8sjpkrh8eahn8u6adqrh4bid20d7ggar9o7uh62pprq5e4kp15me9dkqqjcc3/f3/178e4c55e67d40638256035b48c2f321 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/f3/d0fed44ffc934fbcb597ebd8e11ab07e_SeqId_4_ 2024-12-05T15:49:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413742233/mob/744bacfadce540928e56cdcf38de555b to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__rcp8sjpkrh8eahn8u6adqrh4bid20d7ggar9o7uh62pprq5e4kp15me9dkqqjcc3/mob/744bacfadce540928e56cdcf38de555b 2024-12-05T15:49:05,654 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:05,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48435101, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:05,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:05,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5ddfca 2024-12-05T15:49:05,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__rcp8sjpkrh8eahn8u6adqrh4bid20d7ggar9o7uh62pprq5e4kp15me9dkqqjcc3/mob/744bacfadce540928e56cdcf38de555b as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/mob/5fdd1e2d7968456bbd78bfb86b71d5c6_SeqId_4_ 2024-12-05T15:49:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@662a6de7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,657 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:05,657 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:05,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:05,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:05,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43e3346b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:05,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:05,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:05,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:05,659 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:05,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:05,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae40881, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:05,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:05,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@9af99bc 2024-12-05T15:49:05,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:05,662 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:05,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:05,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:05,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:05,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:05,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:05,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:05,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:05,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815. 2024-12-05T15:49:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:05,680 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__rcp8sjpkrh8eahn8u6adqrh4bid20d7ggar9o7uh62pprq5e4kp15me9dkqqjcc3/f/08aeeeb8bb124367bf6ff13ad871d915 into 8449bae89aa1e699ba056d7b9b3032d3/f as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3/f/1433265d50e14b3dba3552818319cd14_SeqId_4_ - updating store file list. 2024-12-05T15:49:05,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:05,686 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815. 2024-12-05T15:49:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,686 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3/f/1433265d50e14b3dba3552818319cd14_SeqId_4_ into 8449bae89aa1e699ba056d7b9b3032d3/f 2024-12-05T15:49:05,686 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__rcp8sjpkrh8eahn8u6adqrh4bid20d7ggar9o7uh62pprq5e4kp15me9dkqqjcc3/f/08aeeeb8bb124367bf6ff13ad871d915 into 8449bae89aa1e699ba056d7b9b3032d3/f (new location: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3/f/1433265d50e14b3dba3552818319cd14_SeqId_4_) 2024-12-05T15:49:05,686 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__rcp8sjpkrh8eahn8u6adqrh4bid20d7ggar9o7uh62pprq5e4kp15me9dkqqjcc3/f/08aeeeb8bb124367bf6ff13ad871d915 2024-12-05T15:49:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7638c362, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:05,688 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:05,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:05,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:05,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cad9e00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:05,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:05,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,689 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:05,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22a70227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:05,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:05,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@9005dd4 2024-12-05T15:49:05,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:05,691 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d78afa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:05,693 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:05,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:05,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:05,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fcd3f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:05,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:05,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:05,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:05,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:05,694 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:05,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123e91a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:05,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:05,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7aca4369 2024-12-05T15:49:05,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:05,697 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-05T15:49:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:05,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:05,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:05,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-05T15:49:05,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:05,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:05,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:05,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:05,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815. 2024-12-05T15:49:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:05,713 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__rcp8sjpkrh8eahn8u6adqrh4bid20d7ggar9o7uh62pprq5e4kp15me9dkqqjcc3/f3/178e4c55e67d40638256035b48c2f321 into 98789158a6f27fe53f9525b6fc318ac3/f3 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/f3/d0fed44ffc934fbcb597ebd8e11ab07e_SeqId_4_ - updating store file list. 2024-12-05T15:49:05,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:05,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/f3/d0fed44ffc934fbcb597ebd8e11ab07e_SeqId_4_ into 98789158a6f27fe53f9525b6fc318ac3/f3 2024-12-05T15:49:05,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__rcp8sjpkrh8eahn8u6adqrh4bid20d7ggar9o7uh62pprq5e4kp15me9dkqqjcc3/f3/178e4c55e67d40638256035b48c2f321 into 98789158a6f27fe53f9525b6fc318ac3/f3 (new location: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/f3/d0fed44ffc934fbcb597ebd8e11ab07e_SeqId_4_) 2024-12-05T15:49:05,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__rcp8sjpkrh8eahn8u6adqrh4bid20d7ggar9o7uh62pprq5e4kp15me9dkqqjcc3/f3/178e4c55e67d40638256035b48c2f321 2024-12-05T15:49:05,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__rcp8sjpkrh8eahn8u6adqrh4bid20d7ggar9o7uh62pprq5e4kp15me9dkqqjcc3/mob/744bacfadce540928e56cdcf38de555b into 98789158a6f27fe53f9525b6fc318ac3/mob as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/mob/5fdd1e2d7968456bbd78bfb86b71d5c6_SeqId_4_ - updating store file list. 2024-12-05T15:49:05,722 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815. 2024-12-05T15:49:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,722 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:05,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/mob/5fdd1e2d7968456bbd78bfb86b71d5c6_SeqId_4_ into 98789158a6f27fe53f9525b6fc318ac3/mob 2024-12-05T15:49:05,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__rcp8sjpkrh8eahn8u6adqrh4bid20d7ggar9o7uh62pprq5e4kp15me9dkqqjcc3/mob/744bacfadce540928e56cdcf38de555b into 98789158a6f27fe53f9525b6fc318ac3/mob (new location: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/mob/5fdd1e2d7968456bbd78bfb86b71d5c6_SeqId_4_) 2024-12-05T15:49:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__rcp8sjpkrh8eahn8u6adqrh4bid20d7ggar9o7uh62pprq5e4kp15me9dkqqjcc3/mob/744bacfadce540928e56cdcf38de555b 2024-12-05T15:49:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef798c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:05,729 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:05,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:05,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:05,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@332f119b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:05,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:05,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,731 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:05,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1560704c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:05,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:05,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3f81b427 2024-12-05T15:49:05,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:05,733 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-05T15:49:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:05,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:05,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:05,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-05T15:49:05,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:05,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:05,747 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:05,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815. 2024-12-05T15:49:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:05,753 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:49:05,753 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:197) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:05,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:05,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:05,754 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-12-05T15:49:05,754 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-12-05T15:49:05,754 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1733413657238 hdfs://localhost:37233/backupUT/backup_1733413657238/default/test-1733413646382/ 2024-12-05T15:49:05,755 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:37233/backupUT/backup_1733413657238/.backup.manifest 2024-12-05T15:49:05,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3., hostname=5a29627a94df,34815,1733413637643, seqNum=2 , the old value is region=table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3., hostname=5a29627a94df,34815,1733413637643, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=5a29627a94df:34815 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-05T15:49:05,755 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-05T15:49:05,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3., hostname=5a29627a94df,34815,1733413637643, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-05T15:49:05,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3., hostname=5a29627a94df,34815,1733413637643, seqNum=2 from cache 2024-12-05T15:49:05,757 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733413657238/.backup.manifest 2024-12-05T15:49:05,757 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'ns2:test-17334136463821' to 'ns2:table2' from full backup image hdfs://localhost:37233/backupUT/backup_1733413657238/ns2/test-17334136463821 2024-12-05T15:49:05,766 INFO [Time-limited test {}] util.RestoreTool(491): Creating target table 'ns2:table2' 2024-12-05T15:49:05,766 DEBUG [Time-limited test {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:37233/backupUT/backup_1733413657238/ns2/test-17334136463821/archive/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728 2024-12-05T15:49:05,767 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:37233/backupUT/backup_1733413657238/ns2/test-17334136463821/archive/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f in region [hdfs://localhost:37233/backupUT/backup_1733413657238/ns2/test-17334136463821/archive/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728] 2024-12-05T15:49:05,771 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:37233/backupUT/backup_1733413657238/ns2/test-17334136463821/archive/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/1159a028de3844c4b30999226eb32483 first=row0 last=row98 2024-12-05T15:49:05,772 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:49:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:table2 2024-12-05T15:49:05,774 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T15:49:05,775 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 67 2024-12-05T15:49:05,775 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T15:49:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-05T15:49:05,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741978_1154 (size=351) 2024-12-05T15:49:05,867 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'table1', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=5a29627a94df:34815 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 34 more 2024-12-05T15:49:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-05T15:49:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-05T15:49:06,187 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4c2f6298605fad0624d0dec8fedda830, NAME => 'ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:49:06,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741979_1155 (size=37) 2024-12-05T15:49:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-05T15:49:06,598 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:49:06,599 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1722): Closing 4c2f6298605fad0624d0dec8fedda830, disabling compactions & flushes 2024-12-05T15:49:06,599 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. 2024-12-05T15:49:06,599 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. 2024-12-05T15:49:06,599 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. after waiting 0 ms 2024-12-05T15:49:06,599 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. 2024-12-05T15:49:06,599 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1973): Closed ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. 2024-12-05T15:49:06,599 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4c2f6298605fad0624d0dec8fedda830: Waiting for close lock at 1733413746598Disabling compacts and flushes for region at 1733413746598Disabling writes for close at 1733413746599 (+1 ms)Writing region close event to WAL at 1733413746599Closed at 1733413746599 2024-12-05T15:49:06,602 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T15:49:06,603 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1733413746602"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733413746602"}]},"ts":"1733413746602"} 2024-12-05T15:49:06,606 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T15:49:06,607 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T15:49:06,607 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413746607"}]},"ts":"1733413746607"} 2024-12-05T15:49:06,609 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2024-12-05T15:49:06,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=4c2f6298605fad0624d0dec8fedda830, ASSIGN}] 2024-12-05T15:49:06,611 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=4c2f6298605fad0624d0dec8fedda830, ASSIGN 2024-12-05T15:49:06,612 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=4c2f6298605fad0624d0dec8fedda830, ASSIGN; state=OFFLINE, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=false 2024-12-05T15:49:06,758 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-12-05T15:49:06,763 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=4c2f6298605fad0624d0dec8fedda830, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:06,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=4c2f6298605fad0624d0dec8fedda830, ASSIGN because future has completed 2024-12-05T15:49:06,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c2f6298605fad0624d0dec8fedda830, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:49:06,788 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2024-12-05T15:49:06,788 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2024-12-05T15:49:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-05T15:49:06,923 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. 2024-12-05T15:49:06,923 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c2f6298605fad0624d0dec8fedda830, NAME => 'ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830.', STARTKEY => '', ENDKEY => ''} 2024-12-05T15:49:06,923 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:49:06,923 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 4c2f6298605fad0624d0dec8fedda830 2024-12-05T15:49:06,923 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:49:06,923 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for 4c2f6298605fad0624d0dec8fedda830 2024-12-05T15:49:06,923 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for 4c2f6298605fad0624d0dec8fedda830 2024-12-05T15:49:06,925 INFO [StoreOpener-4c2f6298605fad0624d0dec8fedda830-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 4c2f6298605fad0624d0dec8fedda830 2024-12-05T15:49:06,926 INFO [StoreOpener-4c2f6298605fad0624d0dec8fedda830-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c2f6298605fad0624d0dec8fedda830 columnFamilyName f 2024-12-05T15:49:06,926 DEBUG [StoreOpener-4c2f6298605fad0624d0dec8fedda830-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:06,927 INFO [StoreOpener-4c2f6298605fad0624d0dec8fedda830-1 {}] regionserver.HStore(327): Store=4c2f6298605fad0624d0dec8fedda830/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:06,927 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for 4c2f6298605fad0624d0dec8fedda830 2024-12-05T15:49:06,928 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830 2024-12-05T15:49:06,928 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830 2024-12-05T15:49:06,928 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for 4c2f6298605fad0624d0dec8fedda830 2024-12-05T15:49:06,929 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for 4c2f6298605fad0624d0dec8fedda830 2024-12-05T15:49:06,930 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for 4c2f6298605fad0624d0dec8fedda830 2024-12-05T15:49:06,932 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T15:49:06,933 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened 4c2f6298605fad0624d0dec8fedda830; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10115295360, jitterRate=-0.05793970823287964}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T15:49:06,933 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c2f6298605fad0624d0dec8fedda830 2024-12-05T15:49:06,934 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for 4c2f6298605fad0624d0dec8fedda830: Running coprocessor pre-open hook at 1733413746924Writing region info on filesystem at 1733413746924Initializing all the Stores at 1733413746924Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413746924Cleaning up temporary data from old regions at 1733413746929 (+5 ms)Running coprocessor post-open hooks at 1733413746933 (+4 ms)Region opened successfully at 1733413746934 (+1 ms) 2024-12-05T15:49:06,935 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830., pid=69, masterSystemTime=1733413746920 2024-12-05T15:49:06,937 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. 2024-12-05T15:49:06,937 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. 2024-12-05T15:49:06,937 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=4c2f6298605fad0624d0dec8fedda830, regionState=OPEN, openSeqNum=2, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:06,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c2f6298605fad0624d0dec8fedda830, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:49:06,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-05T15:49:06,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c2f6298605fad0624d0dec8fedda830, server=5a29627a94df,34815,1733413637643 in 174 msec 2024-12-05T15:49:06,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-12-05T15:49:06,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=4c2f6298605fad0624d0dec8fedda830, ASSIGN in 333 msec 2024-12-05T15:49:06,945 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T15:49:06,945 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413746945"}]},"ts":"1733413746945"} 2024-12-05T15:49:06,947 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2024-12-05T15:49:06,947 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T15:49:06,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:table2 in 1.1750 sec 2024-12-05T15:49:07,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-05T15:49:07,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:table2 completed 2024-12-05T15:49:07,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-12-05T15:49:07,936 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:37233/backupUT/backup_1733413657238/ns2/test-17334136463821/archive/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728 from hbase tables ns2:test-17334136463821 to tables ns2:table2 2024-12-05T15:49:07,936 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-17334136463821 into ns2:table2 2024-12-05T15:49:07,938 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-ns2-table2-1733413747936 from hdfs://localhost:37233/backupUT/backup_1733413657238/ns2/test-17334136463821/archive/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728 2024-12-05T15:49:07,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6925755e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:07,939 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:07,939 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:07,939 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:07,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:07,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:07,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3e91d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:07,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:07,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:07,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:07,941 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:07,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7624fc0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:07,941 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:07,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:07,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@37a3716d 2024-12-05T15:49:07,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:07,943 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:07,944 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-05T15:49:07,944 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:table2 2024-12-05T15:49:07,944 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-12-05T15:49:07,944 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:07,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:07,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:07,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:07,947 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:table2: 2024-12-05T15:49:07,947 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 1 reduce partitions to match current region count for all tables 2024-12-05T15:49:07,947 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/partitions_cbc15f2a-0ea3-4fca-8b65-1fe78f6109e9 2024-12-05T15:49:07,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741980_1156 (size=153) 2024-12-05T15:49:08,355 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:08,355 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:08,356 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:08,569 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region f422dcfec85c6b145613ea93a68881af, had cached 0 bytes from a total of 41499 2024-12-05T15:49:08,935 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:table2' 2024-12-05T15:49:09,109 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-16766617296390568873.jar 2024-12-05T15:49:09,109 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:09,110 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:09,110 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:09,110 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:09,110 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:09,110 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:09,111 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:09,111 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:09,111 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:09,111 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:49:09,111 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-05T15:49:09,112 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-05T15:49:09,112 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-05T15:49:09,112 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-05T15:49:09,112 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-05T15:49:09,112 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-05T15:49:09,112 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-05T15:49:09,113 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-05T15:49:09,113 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-05T15:49:09,113 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-05T15:49:09,113 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:09,113 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:09,114 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:09,114 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:09,114 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:09,114 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:09,114 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:49:09,114 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:table2 2024-12-05T15:49:09,114 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:49:09,115 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:197) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:09,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:09,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:09,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:09,115 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-12-05T15:49:09,115 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:49:09,124 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:49:09,134 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-12-05T15:49:09,214 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local357659256_0013_5b2771cc-d7a9-4a04-bfe9-fbbf222c6e5f/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-05T15:49:09,214 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local357659256_0013_5b2771cc-d7a9-4a04-bfe9-fbbf222c6e5f/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-05T15:49:09,266 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:37233/backupUT/backup_1733413657238/ns2/test-17334136463821/archive/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/1159a028de3844c4b30999226eb32483 2024-12-05T15:49:09,270 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-05T15:49:09,299 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eedfd2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:09,299 DEBUG [pool-272-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:09,299 DEBUG [pool-272-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:09,300 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:09,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:09,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:09,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9aaabb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:09,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:09,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:09,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:09,302 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:09,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70371a6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:09,304 DEBUG [pool-272-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:09,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:09,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:09,306 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:09,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='row0', locateType=CURRENT is [region=ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:09,308 INFO [pool-272-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-272-thread-1. 2024-12-05T15:49:09,308 DEBUG [pool-272-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:09,309 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:09,309 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:09,309 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:09,309 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row0] 2024-12-05T15:49:09,309 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5a29627a94df 2024-12-05T15:49:09,316 DEBUG [pool-272-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-ns2-table2-1733413747936/_temporary/0/_temporary/attempt_local357659256_0013_r_000000_0/f/60370b9b402a435d891526e557009f5b is 37, key is row10/f:q1/1733413651450/Put/seqid=0 2024-12-05T15:49:09,318 WARN [Thread-1719 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5a29627a94df:34815] Specified favored nodes: [5a29627a94df:34815] 2024-12-05T15:49:09,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741981_1157 (size=8741) 2024-12-05T15:49:10,256 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-ns2-table2-1733413747936 2024-12-05T15:49:10,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1488817b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:10,258 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:10,259 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:10,259 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:10,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:10,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:10,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@153ced9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:10,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:10,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:10,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:10,261 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:10,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c49f7a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:10,262 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:10,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:10,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:10,263 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:10,266 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-ns2-table2-1733413747936/_SUCCESS 2024-12-05T15:49:10,269 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:10,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:10,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54b04d0c 2024-12-05T15:49:10,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:10,272 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:10,276 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:10,278 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-12-05T15:49:10,285 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-ns2-table2-1733413747936/f/60370b9b402a435d891526e557009f5b first=Optional[row0] last=Optional[row98] 2024-12-05T15:49:10,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-ns2-table2-1733413747936/f/60370b9b402a435d891526e557009f5b for inclusion in 4c2f6298605fad0624d0dec8fedda830/f 2024-12-05T15:49:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(626): HFile bounds: first=row0 last=row98 2024-12-05T15:49:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-05T15:49:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HRegion(2603): Flush status journal for 4c2f6298605fad0624d0dec8fedda830: 2024-12-05T15:49:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-ns2-table2-1733413747936/f/60370b9b402a435d891526e557009f5b to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__ns2_table2__alo79ise83cu5dqmfgdbvosruap0eg5lh31s6hc740v83tco8lg4ul9g3lf72559/f/60370b9b402a435d891526e557009f5b 2024-12-05T15:49:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__ns2_table2__alo79ise83cu5dqmfgdbvosruap0eg5lh31s6hc740v83tco8lg4ul9g3lf72559/f/60370b9b402a435d891526e557009f5b as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830/f/28db3e03550e42c8b8390f8ae5f5887f_SeqId_4_ 2024-12-05T15:49:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c1f97b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:10,299 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:10,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:10,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:10,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@242e0854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:10,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:10,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:10,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:10,301 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:10,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@112200d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:10,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:10,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@160a5942 2024-12-05T15:49:10,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:10,303 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:10,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:10,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:10,306 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:10,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:10,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:10,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:10,320 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815. 2024-12-05T15:49:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:10,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:10,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__ns2_table2__alo79ise83cu5dqmfgdbvosruap0eg5lh31s6hc740v83tco8lg4ul9g3lf72559/f/60370b9b402a435d891526e557009f5b into 4c2f6298605fad0624d0dec8fedda830/f as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830/f/28db3e03550e42c8b8390f8ae5f5887f_SeqId_4_ - updating store file list. 2024-12-05T15:49:10,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830/f/28db3e03550e42c8b8390f8ae5f5887f_SeqId_4_ into 4c2f6298605fad0624d0dec8fedda830/f 2024-12-05T15:49:10,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__ns2_table2__alo79ise83cu5dqmfgdbvosruap0eg5lh31s6hc740v83tco8lg4ul9g3lf72559/f/60370b9b402a435d891526e557009f5b into 4c2f6298605fad0624d0dec8fedda830/f (new location: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830/f/28db3e03550e42c8b8390f8ae5f5887f_SeqId_4_) 2024-12-05T15:49:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__ns2_table2__alo79ise83cu5dqmfgdbvosruap0eg5lh31s6hc740v83tco8lg4ul9g3lf72559/f/60370b9b402a435d891526e557009f5b 2024-12-05T15:49:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25c8c1cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:10,329 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:10,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:10,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:10,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49bee889, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:10,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:10,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:10,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:10,331 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:10,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b6d315e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:10,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:10,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4f728ae7 2024-12-05T15:49:10,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:10,333 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-05T15:49:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:10,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:10,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:10,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-05T15:49:10,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:10,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:10,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:10,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815. 2024-12-05T15:49:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:10,352 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:10,352 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:49:10,353 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:197) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:10,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:10,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:10,353 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:10,353 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-12-05T15:49:10,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830., hostname=5a29627a94df,34815,1733413637643, seqNum=2 , the old value is region=ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830., hostname=5a29627a94df,34815,1733413637643, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=5a29627a94df:34815 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-05T15:49:10,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830., hostname=5a29627a94df,34815,1733413637643, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-05T15:49:10,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830., hostname=5a29627a94df,34815,1733413637643, seqNum=2 from cache 2024-12-05T15:49:10,353 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-12-05T15:49:10,353 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1733413657238 hdfs://localhost:37233/backupUT/backup_1733413657238/ns2/test-17334136463821/ 2024-12-05T15:49:10,354 DEBUG [Time-limited test {}] impl.RestoreTablesClient(236): restoreStage finished 2024-12-05T15:49:10,354 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-05T15:49:10,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:10,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:10,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:10,377 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:37233/backupUT/backup_1733413727346/.backup.manifest 2024-12-05T15:49:10,379 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733413727346/.backup.manifest 2024-12-05T15:49:10,380 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:37233/backupUT/backup_1733413692606/.backup.manifest 2024-12-05T15:49:10,381 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733413692606/.backup.manifest 2024-12-05T15:49:10,381 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'test-1733413646382' to 'table1' from full backup image hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382 2024-12-05T15:49:10,383 DEBUG [Time-limited test {}] util.RestoreTool(295): Retrieved descriptor: 'test-1733413646382', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} thru backup_1733413727346 2024-12-05T15:49:10,387 INFO [Time-limited test {}] util.RestoreTool(480): Truncating exising target table 'table1', preserving region splits 2024-12-05T15:49:10,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable table1 2024-12-05T15:49:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=table1 2024-12-05T15:49:10,398 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413750398"}]},"ts":"1733413750398"} 2024-12-05T15:49:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T15:49:10,399 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=DISABLING in hbase:meta 2024-12-05T15:49:10,399 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set table1 to state=DISABLING 2024-12-05T15:49:10,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=table1}] 2024-12-05T15:49:10,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=98789158a6f27fe53f9525b6fc318ac3, UNASSIGN}, {pid=73, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=8449bae89aa1e699ba056d7b9b3032d3, UNASSIGN}] 2024-12-05T15:49:10,406 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=8449bae89aa1e699ba056d7b9b3032d3, UNASSIGN 2024-12-05T15:49:10,406 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=98789158a6f27fe53f9525b6fc318ac3, UNASSIGN 2024-12-05T15:49:10,407 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=8449bae89aa1e699ba056d7b9b3032d3, regionState=CLOSING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:10,407 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=98789158a6f27fe53f9525b6fc318ac3, regionState=CLOSING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:10,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=8449bae89aa1e699ba056d7b9b3032d3, UNASSIGN because future has completed 2024-12-05T15:49:10,408 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T15:49:10,408 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8449bae89aa1e699ba056d7b9b3032d3, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:49:10,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=98789158a6f27fe53f9525b6fc318ac3, UNASSIGN because future has completed 2024-12-05T15:49:10,409 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T15:49:10,409 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure 98789158a6f27fe53f9525b6fc318ac3, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:49:10,457 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'ns2:table2', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=5a29627a94df:34815 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 34 more 2024-12-05T15:49:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T15:49:10,562 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(122): Close 98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:10,562 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T15:49:10,562 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1722): Closing 98789158a6f27fe53f9525b6fc318ac3, disabling compactions & flushes 2024-12-05T15:49:10,562 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1755): Closing region table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. 2024-12-05T15:49:10,563 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. 2024-12-05T15:49:10,563 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1843): Acquired close lock on table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. after waiting 0 ms 2024-12-05T15:49:10,563 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1853): Updates disabled for region table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. 2024-12-05T15:49:10,574 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-12-05T15:49:10,575 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:49:10,576 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1973): Closed table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. 2024-12-05T15:49:10,576 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1676): Region close journal for 98789158a6f27fe53f9525b6fc318ac3: Waiting for close lock at 1733413750562Running coprocessor pre-close hooks at 1733413750562Disabling compacts and flushes for region at 1733413750562Disabling writes for close at 1733413750563 (+1 ms)Writing region close event to WAL at 1733413750565 (+2 ms)Running coprocessor post-close hooks at 1733413750575 (+10 ms)Closed at 1733413750576 (+1 ms) 2024-12-05T15:49:10,579 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(157): Closed 98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:10,579 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] handler.UnassignRegionHandler(122): Close 8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:10,579 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T15:49:10,579 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1722): Closing 8449bae89aa1e699ba056d7b9b3032d3, disabling compactions & flushes 2024-12-05T15:49:10,579 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1755): Closing region table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. 2024-12-05T15:49:10,580 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. 2024-12-05T15:49:10,580 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. after waiting 0 ms 2024-12-05T15:49:10,580 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. 2024-12-05T15:49:10,580 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=98789158a6f27fe53f9525b6fc318ac3, regionState=CLOSED 2024-12-05T15:49:10,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure 98789158a6f27fe53f9525b6fc318ac3, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:49:10,586 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-12-05T15:49:10,586 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=72 2024-12-05T15:49:10,586 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=72, state=SUCCESS, hasLock=false; CloseRegionProcedure 98789158a6f27fe53f9525b6fc318ac3, server=5a29627a94df,34815,1733413637643 in 174 msec 2024-12-05T15:49:10,587 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:49:10,587 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1973): Closed table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. 2024-12-05T15:49:10,587 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1676): Region close journal for 8449bae89aa1e699ba056d7b9b3032d3: Waiting for close lock at 1733413750579Running coprocessor pre-close hooks at 1733413750579Disabling compacts and flushes for region at 1733413750579Disabling writes for close at 1733413750580 (+1 ms)Writing region close event to WAL at 1733413750581 (+1 ms)Running coprocessor post-close hooks at 1733413750586 (+5 ms)Closed at 1733413750587 (+1 ms) 2024-12-05T15:49:10,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=98789158a6f27fe53f9525b6fc318ac3, UNASSIGN in 181 msec 2024-12-05T15:49:10,589 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] handler.UnassignRegionHandler(157): Closed 8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:10,590 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=8449bae89aa1e699ba056d7b9b3032d3, regionState=CLOSED 2024-12-05T15:49:10,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=73, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8449bae89aa1e699ba056d7b9b3032d3, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:49:10,593 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-12-05T15:49:10,593 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; CloseRegionProcedure 8449bae89aa1e699ba056d7b9b3032d3, server=5a29627a94df,34815,1733413637643 in 184 msec 2024-12-05T15:49:10,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=71 2024-12-05T15:49:10,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=71, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=8449bae89aa1e699ba056d7b9b3032d3, UNASSIGN in 188 msec 2024-12-05T15:49:10,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-05T15:49:10,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=table1 in 194 msec 2024-12-05T15:49:10,598 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413750598"}]},"ts":"1733413750598"} 2024-12-05T15:49:10,599 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=DISABLED in hbase:meta 2024-12-05T15:49:10,599 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set table1 to state=DISABLED 2024-12-05T15:49:10,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; DisableTableProcedure table=table1 in 208 msec 2024-12-05T15:49:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T15:49:10,717 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:table1 completed 2024-12-05T15:49:10,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.HMaster$6(2602): Client=jenkins//172.17.0.3 truncate table1 2024-12-05T15:49:10,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION, hasLock=false; TruncateTableProcedure (table=table1 preserveSplits=true) 2024-12-05T15:49:10,735 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(87): waiting for 'table1' regions in transition 2024-12-05T15:49:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T15:49:10,740 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:10,740 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:10,743 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/f, FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/f3, FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/mob, FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/recovered.edits] 2024-12-05T15:49:10,743 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3/f, FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3/f3, FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3/mob, FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3/recovered.edits] 2024-12-05T15:49:10,748 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3/f/1433265d50e14b3dba3552818319cd14_SeqId_4_ to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/archive/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3/f/1433265d50e14b3dba3552818319cd14_SeqId_4_ 2024-12-05T15:49:10,749 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/f3/d0fed44ffc934fbcb597ebd8e11ab07e_SeqId_4_ to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/archive/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/f3/d0fed44ffc934fbcb597ebd8e11ab07e_SeqId_4_ 2024-12-05T15:49:10,752 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/mob/5fdd1e2d7968456bbd78bfb86b71d5c6_SeqId_4_ to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/archive/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/mob/5fdd1e2d7968456bbd78bfb86b71d5c6_SeqId_4_ 2024-12-05T15:49:10,753 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3/recovered.edits/6.seqid to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/archive/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3/recovered.edits/6.seqid 2024-12-05T15:49:10,754 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/8449bae89aa1e699ba056d7b9b3032d3 2024-12-05T15:49:10,755 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/recovered.edits/6.seqid to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/archive/data/default/table1/98789158a6f27fe53f9525b6fc318ac3/recovered.edits/6.seqid 2024-12-05T15:49:10,755 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/98789158a6f27fe53f9525b6fc318ac3 2024-12-05T15:49:10,755 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived table1 regions 2024-12-05T15:49:10,756 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/mobdir/data/default/table1/b4aaa492ee713125ae5609f2682a2d2f 2024-12-05T15:49:10,757 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/mobdir/data/default/table1/b4aaa492ee713125ae5609f2682a2d2f/mob] 2024-12-05T15:49:10,759 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/mobdir/data/default/table1/b4aaa492ee713125ae5609f2682a2d2f 2024-12-05T15:49:10,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34815 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-05T15:49:10,769 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 2 rows of table1 from hbase:meta 2024-12-05T15:49:10,772 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'table1' descriptor. 2024-12-05T15:49:10,772 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'table1' from region states. 2024-12-05T15:49:10,773 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733413750773"}]},"ts":"9223372036854775807"} 2024-12-05T15:49:10,773 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733413750773"}]},"ts":"9223372036854775807"} 2024-12-05T15:49:10,775 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 2 regions from META 2024-12-05T15:49:10,775 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 98789158a6f27fe53f9525b6fc318ac3, NAME => 'table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3.', STARTKEY => '', ENDKEY => 'row0'}, {ENCODED => 8449bae89aa1e699ba056d7b9b3032d3, NAME => 'table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3.', STARTKEY => 'row0', ENDKEY => ''}] 2024-12-05T15:49:10,776 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'table1' as deleted. 2024-12-05T15:49:10,776 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733413750776"}]},"ts":"9223372036854775807"} 2024-12-05T15:49:10,778 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table table1 state from META 2024-12-05T15:49:10,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741982_1158 (size=938) 2024-12-05T15:49:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T15:49:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T15:49:11,192 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 003f376bc19761c728c4bc8dd20d416d, NAME => 'table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d.', STARTKEY => '', ENDKEY => 'row0'}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:49:11,193 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 694967a9c2dfe786b3822f9f145f9365, NAME => 'table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365.', STARTKEY => 'row0', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:49:11,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741983_1159 (size=45) 2024-12-05T15:49:11,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741984_1160 (size=45) 2024-12-05T15:49:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T15:49:11,604 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(898): Instantiated table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:49:11,604 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:49:11,604 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1722): Closing 694967a9c2dfe786b3822f9f145f9365, disabling compactions & flushes 2024-12-05T15:49:11,604 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1755): Closing region table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:11,604 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing 003f376bc19761c728c4bc8dd20d416d, disabling compactions & flushes 2024-12-05T15:49:11,604 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:11,604 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:11,605 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. after waiting 0 ms 2024-12-05T15:49:11,605 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:11,605 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:11,605 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. after waiting 0 ms 2024-12-05T15:49:11,605 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1973): Closed table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:11,605 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:11,605 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:11,605 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1676): Region close journal for 694967a9c2dfe786b3822f9f145f9365: Waiting for close lock at 1733413751604Disabling compacts and flushes for region at 1733413751604Disabling writes for close at 1733413751605 (+1 ms)Writing region close event to WAL at 1733413751605Closed at 1733413751605 2024-12-05T15:49:11,605 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for 003f376bc19761c728c4bc8dd20d416d: Waiting for close lock at 1733413751604Disabling compacts and flushes for region at 1733413751604Disabling writes for close at 1733413751605 (+1 ms)Writing region close event to WAL at 1733413751605Closed at 1733413751605 2024-12-05T15:49:11,609 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733413751608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733413751608"}]},"ts":"1733413751608"} 2024-12-05T15:49:11,609 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733413751608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733413751608"}]},"ts":"1733413751608"} 2024-12-05T15:49:11,614 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 2 regions to meta. 2024-12-05T15:49:11,615 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413751615"}]},"ts":"1733413751615"} 2024-12-05T15:49:11,618 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-12-05T15:49:11,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=003f376bc19761c728c4bc8dd20d416d, ASSIGN}, {pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=694967a9c2dfe786b3822f9f145f9365, ASSIGN}] 2024-12-05T15:49:11,620 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=003f376bc19761c728c4bc8dd20d416d, ASSIGN 2024-12-05T15:49:11,620 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=694967a9c2dfe786b3822f9f145f9365, ASSIGN 2024-12-05T15:49:11,621 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=003f376bc19761c728c4bc8dd20d416d, ASSIGN; state=OFFLINE, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=false 2024-12-05T15:49:11,621 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=694967a9c2dfe786b3822f9f145f9365, ASSIGN; state=OFFLINE, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=false 2024-12-05T15:49:11,772 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=694967a9c2dfe786b3822f9f145f9365, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:11,772 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=77 updating hbase:meta row=003f376bc19761c728c4bc8dd20d416d, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:11,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=003f376bc19761c728c4bc8dd20d416d, ASSIGN because future has completed 2024-12-05T15:49:11,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=77, state=RUNNABLE, hasLock=false; OpenRegionProcedure 003f376bc19761c728c4bc8dd20d416d, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:49:11,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=694967a9c2dfe786b3822f9f145f9365, ASSIGN because future has completed 2024-12-05T15:49:11,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure 694967a9c2dfe786b3822f9f145f9365, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:49:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T15:49:11,938 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:11,939 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 694967a9c2dfe786b3822f9f145f9365, NAME => 'table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365.', STARTKEY => 'row0', ENDKEY => ''} 2024-12-05T15:49:11,939 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:49:11,940 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:11,940 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:49:11,940 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:11,940 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:11,943 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:11,945 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 694967a9c2dfe786b3822f9f145f9365 columnFamilyName f 2024-12-05T15:49:11,945 DEBUG [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:11,946 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] regionserver.HStore(327): Store=694967a9c2dfe786b3822f9f145f9365/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:11,946 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:11,948 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 694967a9c2dfe786b3822f9f145f9365 columnFamilyName f3 2024-12-05T15:49:11,948 DEBUG [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:11,949 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] regionserver.HStore(327): Store=694967a9c2dfe786b3822f9f145f9365/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:11,949 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:11,951 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 694967a9c2dfe786b3822f9f145f9365 columnFamilyName mob 2024-12-05T15:49:11,953 DEBUG [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:11,954 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] regionserver.HStore(327): Store=694967a9c2dfe786b3822f9f145f9365/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:11,955 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:11,956 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:11,957 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:11,959 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:11,959 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:11,960 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T15:49:11,961 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:11,963 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/694967a9c2dfe786b3822f9f145f9365/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T15:49:11,964 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 694967a9c2dfe786b3822f9f145f9365; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9929723680, jitterRate=-0.07522241771221161}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T15:49:11,964 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:11,964 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 694967a9c2dfe786b3822f9f145f9365: Running coprocessor pre-open hook at 1733413751940Writing region info on filesystem at 1733413751940Initializing all the Stores at 1733413751942 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413751942Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413751943 (+1 ms)Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413751943Cleaning up temporary data from old regions at 1733413751959 (+16 ms)Running coprocessor post-open hooks at 1733413751964 (+5 ms)Region opened successfully at 1733413751964 2024-12-05T15:49:11,965 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365., pid=80, masterSystemTime=1733413751932 2024-12-05T15:49:11,967 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:11,967 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:11,967 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:11,967 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 003f376bc19761c728c4bc8dd20d416d, NAME => 'table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d.', STARTKEY => '', ENDKEY => 'row0'} 2024-12-05T15:49:11,967 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=694967a9c2dfe786b3822f9f145f9365, regionState=OPEN, openSeqNum=2, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:11,967 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:49:11,968 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:11,968 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:49:11,968 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:11,968 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:11,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure 694967a9c2dfe786b3822f9f145f9365, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:49:11,969 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:11,970 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 003f376bc19761c728c4bc8dd20d416d columnFamilyName f 2024-12-05T15:49:11,970 DEBUG [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:11,971 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] regionserver.HStore(327): Store=003f376bc19761c728c4bc8dd20d416d/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:11,971 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:11,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=78 2024-12-05T15:49:11,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=78, state=SUCCESS, hasLock=false; OpenRegionProcedure 694967a9c2dfe786b3822f9f145f9365, server=5a29627a94df,34815,1733413637643 in 190 msec 2024-12-05T15:49:11,972 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 003f376bc19761c728c4bc8dd20d416d columnFamilyName f3 2024-12-05T15:49:11,972 DEBUG [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:11,973 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] regionserver.HStore(327): Store=003f376bc19761c728c4bc8dd20d416d/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:11,973 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:11,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=76, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=694967a9c2dfe786b3822f9f145f9365, ASSIGN in 354 msec 2024-12-05T15:49:11,974 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 003f376bc19761c728c4bc8dd20d416d columnFamilyName mob 2024-12-05T15:49:11,974 DEBUG [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:11,975 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] regionserver.HStore(327): Store=003f376bc19761c728c4bc8dd20d416d/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:11,975 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:11,975 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:11,976 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:11,977 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:11,977 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:11,978 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T15:49:11,979 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:11,980 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T15:49:11,981 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 003f376bc19761c728c4bc8dd20d416d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10562790560, jitterRate=-0.01626347005367279}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T15:49:11,981 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:11,981 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 003f376bc19761c728c4bc8dd20d416d: Running coprocessor pre-open hook at 1733413751968Writing region info on filesystem at 1733413751968Initializing all the Stores at 1733413751969 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413751969Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413751969Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413751969Cleaning up temporary data from old regions at 1733413751977 (+8 ms)Running coprocessor post-open hooks at 1733413751981 (+4 ms)Region opened successfully at 1733413751981 2024-12-05T15:49:11,982 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., pid=79, masterSystemTime=1733413751932 2024-12-05T15:49:11,983 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:11,984 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:11,984 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=77 updating hbase:meta row=003f376bc19761c728c4bc8dd20d416d, regionState=OPEN, openSeqNum=2, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:11,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=77, state=RUNNABLE, hasLock=false; OpenRegionProcedure 003f376bc19761c728c4bc8dd20d416d, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:49:11,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=77 2024-12-05T15:49:11,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=77, state=SUCCESS, hasLock=false; OpenRegionProcedure 003f376bc19761c728c4bc8dd20d416d, server=5a29627a94df,34815,1733413637643 in 209 msec 2024-12-05T15:49:11,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-05T15:49:11,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=003f376bc19761c728c4bc8dd20d416d, ASSIGN in 370 msec 2024-12-05T15:49:11,989 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413751989"}]},"ts":"1733413751989"} 2024-12-05T15:49:11,991 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-12-05T15:49:11,991 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(145): truncate 'table1' completed 2024-12-05T15:49:11,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; TruncateTableProcedure (table=table1 preserveSplits=true) in 1.2610 sec 2024-12-05T15:49:12,427 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-12-05T15:49:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T15:49:12,898 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: TRUNCATE, Table Name: default:table1 completed 2024-12-05T15:49:12,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-05T15:49:12,910 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0,hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/archive/data/default/test-1733413646382/d269731e9ae3c5ae370e1788a2e183c3 from hbase tables test-1733413646382 to tables table1 2024-12-05T15:49:12,910 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore test-1733413646382 into table1 2024-12-05T15:49:12,912 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413752910 from hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0,hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/archive/data/default/test-1733413646382/d269731e9ae3c5ae370e1788a2e183c3 2024-12-05T15:49:12,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71ffa5e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:12,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:12,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:12,914 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:12,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:12,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:12,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7249d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:12,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:12,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:12,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:12,916 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:12,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@747f2d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:12,917 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:12,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:12,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7347623e 2024-12-05T15:49:12,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:12,919 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:12,920 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-05T15:49:12,920 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table table1 2024-12-05T15:49:12,920 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-05T15:49:12,920 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:12,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:12,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:12,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:12,925 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: 2024-12-05T15:49:12,925 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: row0 2024-12-05T15:49:12,925 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-12-05T15:49:12,925 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/partitions_82d60a22-5ae9-4be3-bc42-dacf05ff4741 2024-12-05T15:49:12,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741985_1161 (size=177) 2024-12-05T15:49:13,336 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:13,337 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:13,338 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-9725006101400628052.jar 2024-12-05T15:49:14,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,100 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,100 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,100 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,100 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,101 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,101 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,101 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,101 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:49:14,101 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-05T15:49:14,102 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-05T15:49:14,102 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-05T15:49:14,102 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-05T15:49:14,102 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-05T15:49:14,102 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-05T15:49:14,102 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-05T15:49:14,103 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-05T15:49:14,103 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-05T15:49:14,103 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-05T15:49:14,103 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,103 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,104 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,104 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,104 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,104 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:14,104 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:49:14,104 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: table1 2024-12-05T15:49:14,104 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:49:14,105 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:218) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:14,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:14,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:14,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:14,105 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-12-05T15:49:14,105 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:49:14,114 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:49:14,125 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-12-05T15:49:14,215 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local941707472_0014_2fe81861-1b13-4b0d-b421-72834f557d8f/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-05T15:49:14,215 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local941707472_0014_2fe81861-1b13-4b0d-b421-72834f557d8f/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-05T15:49:14,264 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/d940ad880da148ee8bac344a248bd8c3 2024-12-05T15:49:14,268 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-05T15:49:14,309 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/91dfff1e2dee4ba59bbdb059b1d0570f 2024-12-05T15:49:14,312 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-05T15:49:14,335 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f/4e364da1714b48f9af4efc97be4888c8 2024-12-05T15:49:14,337 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-05T15:49:14,369 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/archive/data/default/test-1733413646382/d269731e9ae3c5ae370e1788a2e183c3/mob/d41d8cd98f00b204e9800998ecf8427e20241205070e4aa4b5934388ad16285a00c5ad73_24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:49:14,372 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-05T15:49:14,392 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:37233/backupUT/backup_1733413692606/default/test-1733413646382/archive/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/mob/09842ffb0751416eaaeffe3bff7aefa1 2024-12-05T15:49:14,395 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-05T15:49:14,427 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f054c0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:14,427 DEBUG [pool-277-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:14,427 DEBUG [pool-277-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:14,428 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:14,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:14,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:14,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f3d9f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:14,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:14,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:14,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:14,431 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:14,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29dd5a72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:14,432 DEBUG [pool-277-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:14,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:14,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:14,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:14,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1733413646382-1-0', locateType=CURRENT is [region=table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:14,437 INFO [pool-277-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-277-thread-1. 2024-12-05T15:49:14,437 DEBUG [pool-277-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:14,437 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:14,437 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:14,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:14,437 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733413646382-1-0] 2024-12-05T15:49:14,437 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5a29627a94df 2024-12-05T15:49:14,447 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1add6344, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:14,447 DEBUG [pool-277-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:14,447 DEBUG [pool-277-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:14,448 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:14,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:14,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:14,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ffdbee6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:14,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:14,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:14,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:14,449 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:14,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@506813a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:14,450 DEBUG [pool-277-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:14,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:14,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:14,452 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:14,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1733413646382-3-0', locateType=CURRENT is [region=table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:14,454 INFO [pool-277-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-277-thread-1. 2024-12-05T15:49:14,454 DEBUG [pool-277-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:14,454 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:14,454 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:14,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:14,454 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733413646382-3-0] 2024-12-05T15:49:14,454 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5a29627a94df 2024-12-05T15:49:14,463 DEBUG [pool-277-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413752910/_temporary/0/_temporary/attempt_local941707472_0014_r_000000_0/f/a4148b7c237d43328c0c78252ffe7c2b is 59, key is row-test-1733413646382-1-10/f:q1/1733413677257/Put/seqid=0 2024-12-05T15:49:14,465 WARN [Thread-1806 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5a29627a94df:34815] Specified favored nodes: [5a29627a94df:34815] 2024-12-05T15:49:14,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741986_1162 (size=11124) 2024-12-05T15:49:14,869 DEBUG [pool-277-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413752910/_temporary/0/_temporary/attempt_local941707472_0014_r_000000_0/mob/61fde137997949e0be32aa40ea2821a6 is 192, key is row-test-1733413646382-3-100/mob:q1/1733413677609/Put/seqid=0 2024-12-05T15:49:14,872 WARN [Thread-1808 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5a29627a94df:34815] Specified favored nodes: [5a29627a94df:34815] 2024-12-05T15:49:14,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741987_1163 (size=13403) 2024-12-05T15:49:15,267 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T15:49:15,309 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25048222, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:15,309 DEBUG [pool-277-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:15,309 DEBUG [pool-277-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:15,309 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:15,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:15,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:15,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26d80cbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:15,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:15,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:15,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:15,311 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:15,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ffd8fde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:15,312 DEBUG [pool-277-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:15,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:15,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:15,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:15,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:15,316 INFO [pool-277-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-277-thread-1. 2024-12-05T15:49:15,316 DEBUG [pool-277-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:15,316 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:15,317 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:15,317 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:15,317 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row0] 2024-12-05T15:49:15,317 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5a29627a94df 2024-12-05T15:49:15,323 DEBUG [pool-277-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413752910/_temporary/0/_temporary/attempt_local941707472_0014_r_000001_0/f/336aae6d2ba44d9b94c88c60239825fa is 37, key is row10/f:q1/1733413649096/Put/seqid=0 2024-12-05T15:49:15,325 WARN [Thread-1823 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5a29627a94df:34815] Specified favored nodes: [5a29627a94df:34815] 2024-12-05T15:49:15,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741988_1164 (size=8741) 2024-12-05T15:49:16,262 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413752910 2024-12-05T15:49:16,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d3ecdc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:16,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:16,264 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:16,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:16,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:16,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa75ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:16,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:16,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,265 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:16,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae6c5eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,266 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:16,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:16,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:16,268 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:16,270 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413752910/_SUCCESS 2024-12-05T15:49:16,277 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:16,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:16,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@606a6f9b 2024-12-05T15:49:16,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:16,280 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:16,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:16,287 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-05T15:49:16,293 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413752910/f/336aae6d2ba44d9b94c88c60239825fa first=Optional[row0] last=Optional[row98] 2024-12-05T15:49:16,296 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413752910/f/a4148b7c237d43328c0c78252ffe7c2b first=Optional[row-test-1733413646382-1-0] last=Optional[row-test-1733413646382-1-98] 2024-12-05T15:49:16,299 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413752910/mob/61fde137997949e0be32aa40ea2821a6 first=Optional[row-test-1733413646382-3-0] last=Optional[row-test-1733413646382-3-99] 2024-12-05T15:49:16,303 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413752910/f/336aae6d2ba44d9b94c88c60239825fa for inclusion in 694967a9c2dfe786b3822f9f145f9365/f 2024-12-05T15:49:16,304 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413752910/f/a4148b7c237d43328c0c78252ffe7c2b for inclusion in 003f376bc19761c728c4bc8dd20d416d/f 2024-12-05T15:49:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HStore(626): HFile bounds: first=row0 last=row98 2024-12-05T15:49:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HStore(628): Region bounds: first=row0 last= 2024-12-05T15:49:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HRegion(2603): Flush status journal for 694967a9c2dfe786b3822f9f145f9365: 2024-12-05T15:49:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(626): HFile bounds: first=row-test-1733413646382-1-0 last=row-test-1733413646382-1-98 2024-12-05T15:49:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-12-05T15:49:16,308 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413752910/mob/61fde137997949e0be32aa40ea2821a6 for inclusion in 003f376bc19761c728c4bc8dd20d416d/mob 2024-12-05T15:49:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413752910/f/336aae6d2ba44d9b94c88c60239825fa to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__aet2rsfrt14k20kenmoh3drl5v7r3nimnp4frmges8o54tikuf1tgh626ul8iqhl/f/336aae6d2ba44d9b94c88c60239825fa 2024-12-05T15:49:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__aet2rsfrt14k20kenmoh3drl5v7r3nimnp4frmges8o54tikuf1tgh626ul8iqhl/f/336aae6d2ba44d9b94c88c60239825fa as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/694967a9c2dfe786b3822f9f145f9365/f/8dded74aa4a74b06b672b010a60967af_SeqId_4_ 2024-12-05T15:49:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(626): HFile bounds: first=row-test-1733413646382-3-0 last=row-test-1733413646382-3-99 2024-12-05T15:49:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-12-05T15:49:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108627b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HRegion(2603): Flush status journal for 003f376bc19761c728c4bc8dd20d416d: 2024-12-05T15:49:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:16,313 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413752910/f/a4148b7c237d43328c0c78252ffe7c2b to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__aet2rsfrt14k20kenmoh3drl5v7r3nimnp4frmges8o54tikuf1tgh626ul8iqhl/f/a4148b7c237d43328c0c78252ffe7c2b 2024-12-05T15:49:16,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:16,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:16,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4562a5bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:16,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:16,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,315 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:16,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@579bd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__aet2rsfrt14k20kenmoh3drl5v7r3nimnp4frmges8o54tikuf1tgh626ul8iqhl/f/a4148b7c237d43328c0c78252ffe7c2b as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/f/54efbd6b92294e2db01b4d8b1f627d99_SeqId_4_ 2024-12-05T15:49:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413752910/mob/61fde137997949e0be32aa40ea2821a6 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__aet2rsfrt14k20kenmoh3drl5v7r3nimnp4frmges8o54tikuf1tgh626ul8iqhl/mob/61fde137997949e0be32aa40ea2821a6 2024-12-05T15:49:16,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:16,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@36f85b40 2024-12-05T15:49:16,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:16,318 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__aet2rsfrt14k20kenmoh3drl5v7r3nimnp4frmges8o54tikuf1tgh626ul8iqhl/mob/61fde137997949e0be32aa40ea2821a6 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/mob/025555ef841d482f93ea7181a613a078_SeqId_4_ 2024-12-05T15:49:16,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36d2efc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:16,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:16,321 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:16,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:16,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:16,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cea0ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:16,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:16,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:16,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,323 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:16,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7c25fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:16,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:16,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5140db4a 2024-12-05T15:49:16,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:16,325 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:16,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:16,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:16,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:16,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:16,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:16,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:16,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:16,342 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815. 2024-12-05T15:49:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,343 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:16,344 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__aet2rsfrt14k20kenmoh3drl5v7r3nimnp4frmges8o54tikuf1tgh626ul8iqhl/f/336aae6d2ba44d9b94c88c60239825fa into 694967a9c2dfe786b3822f9f145f9365/f as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/694967a9c2dfe786b3822f9f145f9365/f/8dded74aa4a74b06b672b010a60967af_SeqId_4_ - updating store file list. 2024-12-05T15:49:16,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:16,351 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/694967a9c2dfe786b3822f9f145f9365/f/8dded74aa4a74b06b672b010a60967af_SeqId_4_ into 694967a9c2dfe786b3822f9f145f9365/f 2024-12-05T15:49:16,351 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__aet2rsfrt14k20kenmoh3drl5v7r3nimnp4frmges8o54tikuf1tgh626ul8iqhl/f/336aae6d2ba44d9b94c88c60239825fa into 694967a9c2dfe786b3822f9f145f9365/f (new location: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/694967a9c2dfe786b3822f9f145f9365/f/8dded74aa4a74b06b672b010a60967af_SeqId_4_) 2024-12-05T15:49:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__aet2rsfrt14k20kenmoh3drl5v7r3nimnp4frmges8o54tikuf1tgh626ul8iqhl/f/336aae6d2ba44d9b94c88c60239825fa 2024-12-05T15:49:16,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8226e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:16,354 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:16,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:16,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:16,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47453f26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:16,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:16,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815. 2024-12-05T15:49:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,356 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:16,356 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:16,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a7247a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@291f062a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:16,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:16,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2654f50 2024-12-05T15:49:16,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:16,358 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:16,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:16,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:16,359 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-05T15:49:16,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cd7ac90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:16,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:16,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:16,360 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:16,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7342d91e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:16,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:16,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:16,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:16,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4461b146 2024-12-05T15:49:16,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:16,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-05T15:49:16,364 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:16,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:16,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:16,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:16,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:16,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:16,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:16,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:16,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:16,383 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815. 2024-12-05T15:49:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,384 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:16,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:16,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815. 2024-12-05T15:49:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:16,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__aet2rsfrt14k20kenmoh3drl5v7r3nimnp4frmges8o54tikuf1tgh626ul8iqhl/f/a4148b7c237d43328c0c78252ffe7c2b into 003f376bc19761c728c4bc8dd20d416d/f as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/f/54efbd6b92294e2db01b4d8b1f627d99_SeqId_4_ - updating store file list. 2024-12-05T15:49:16,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/f/54efbd6b92294e2db01b4d8b1f627d99_SeqId_4_ into 003f376bc19761c728c4bc8dd20d416d/f 2024-12-05T15:49:16,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__aet2rsfrt14k20kenmoh3drl5v7r3nimnp4frmges8o54tikuf1tgh626ul8iqhl/f/a4148b7c237d43328c0c78252ffe7c2b into 003f376bc19761c728c4bc8dd20d416d/f (new location: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/f/54efbd6b92294e2db01b4d8b1f627d99_SeqId_4_) 2024-12-05T15:49:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__aet2rsfrt14k20kenmoh3drl5v7r3nimnp4frmges8o54tikuf1tgh626ul8iqhl/f/a4148b7c237d43328c0c78252ffe7c2b 2024-12-05T15:49:16,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__aet2rsfrt14k20kenmoh3drl5v7r3nimnp4frmges8o54tikuf1tgh626ul8iqhl/mob/61fde137997949e0be32aa40ea2821a6 into 003f376bc19761c728c4bc8dd20d416d/mob as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/mob/025555ef841d482f93ea7181a613a078_SeqId_4_ - updating store file list. 2024-12-05T15:49:16,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/mob/025555ef841d482f93ea7181a613a078_SeqId_4_ into 003f376bc19761c728c4bc8dd20d416d/mob 2024-12-05T15:49:16,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__aet2rsfrt14k20kenmoh3drl5v7r3nimnp4frmges8o54tikuf1tgh626ul8iqhl/mob/61fde137997949e0be32aa40ea2821a6 into 003f376bc19761c728c4bc8dd20d416d/mob (new location: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/mob/025555ef841d482f93ea7181a613a078_SeqId_4_) 2024-12-05T15:49:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__aet2rsfrt14k20kenmoh3drl5v7r3nimnp4frmges8o54tikuf1tgh626ul8iqhl/mob/61fde137997949e0be32aa40ea2821a6 2024-12-05T15:49:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b880ca8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:16,408 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:16,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:16,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:16,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2687fad0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:16,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:16,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,411 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:16,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b86173a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:16,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:16,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7aab0156 2024-12-05T15:49:16,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:16,414 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-05T15:49:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:16,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:16,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:16,417 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-05T15:49:16,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:16,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:16,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:16,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815. 2024-12-05T15:49:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:16,444 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:49:16,444 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:218) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:16,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:16,444 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-12-05T15:49:16,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:16,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., hostname=5a29627a94df,34815,1733413637643, seqNum=2 , the old value is region=table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., hostname=5a29627a94df,34815,1733413637643, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=5a29627a94df:34815 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-05T15:49:16,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., hostname=5a29627a94df,34815,1733413637643, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-05T15:49:16,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., hostname=5a29627a94df,34815,1733413637643, seqNum=2 from cache 2024-12-05T15:49:16,445 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-05T15:49:16,472 INFO [Time-limited test {}] impl.RestoreTablesClient(181): Restoring 'test-1733413646382' to 'table1' from log dirs: hdfs://localhost:37233/backupUT/backup_1733413727346/default/test-1733413646382/f2/f294e4172be34af5990ccf7377ca6be0 2024-12-05T15:49:16,477 DEBUG [Time-limited test {}] util.RestoreTool(177): Found descriptor 'test-1733413646382', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} through backup_1733413727346 2024-12-05T15:49:16,481 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table table1 from 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:49:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=table1 2024-12-05T15:49:16,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741989_1165 (size=938) 2024-12-05T15:49:16,557 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'table1', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=5a29627a94df:34815 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 34 more 2024-12-05T15:49:16,572 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2024-12-05T15:49:16,572 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_table2 Metrics about Tables on a single HBase RegionServer 2024-12-05T15:49:16,900 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/.tabledesc/.tableinfo.0000000001.938 2024-12-05T15:49:16,900 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/.tabledesc/.tableinfo.0000000002.938 2024-12-05T15:49:16,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=table1}] 2024-12-05T15:49:16,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=003f376bc19761c728c4bc8dd20d416d, REOPEN/MOVE}, {pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=694967a9c2dfe786b3822f9f145f9365, REOPEN/MOVE}] 2024-12-05T15:49:16,910 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=694967a9c2dfe786b3822f9f145f9365, REOPEN/MOVE 2024-12-05T15:49:16,910 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=003f376bc19761c728c4bc8dd20d416d, REOPEN/MOVE 2024-12-05T15:49:16,911 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=003f376bc19761c728c4bc8dd20d416d, regionState=CLOSING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:16,911 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=694967a9c2dfe786b3822f9f145f9365, regionState=CLOSING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:16,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=694967a9c2dfe786b3822f9f145f9365, REOPEN/MOVE because future has completed 2024-12-05T15:49:16,913 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T15:49:16,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 694967a9c2dfe786b3822f9f145f9365, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:49:16,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=003f376bc19761c728c4bc8dd20d416d, REOPEN/MOVE because future has completed 2024-12-05T15:49:16,914 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T15:49:16,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=83, state=RUNNABLE, hasLock=false; CloseRegionProcedure 003f376bc19761c728c4bc8dd20d416d, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:49:17,067 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] handler.UnassignRegionHandler(122): Close 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:17,067 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T15:49:17,067 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1722): Closing 003f376bc19761c728c4bc8dd20d416d, disabling compactions & flushes 2024-12-05T15:49:17,068 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1755): Closing region table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:17,068 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:17,068 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1843): Acquired close lock on table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. after waiting 0 ms 2024-12-05T15:49:17,068 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1853): Updates disabled for region table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:17,079 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-12-05T15:49:17,080 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:49:17,080 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1973): Closed table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:17,080 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1676): Region close journal for 003f376bc19761c728c4bc8dd20d416d: Waiting for close lock at 1733413757067Running coprocessor pre-close hooks at 1733413757067Disabling compacts and flushes for region at 1733413757067Disabling writes for close at 1733413757068 (+1 ms)Writing region close event to WAL at 1733413757070 (+2 ms)Running coprocessor post-close hooks at 1733413757080 (+10 ms)Closed at 1733413757080 2024-12-05T15:49:17,080 WARN [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegionServer(3299): Not adding moved region record: 003f376bc19761c728c4bc8dd20d416d to self. 2024-12-05T15:49:17,083 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] handler.UnassignRegionHandler(157): Closed 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:17,084 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:17,084 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T15:49:17,084 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 694967a9c2dfe786b3822f9f145f9365, disabling compactions & flushes 2024-12-05T15:49:17,084 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:17,084 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:17,084 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. after waiting 0 ms 2024-12-05T15:49:17,084 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:17,084 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=003f376bc19761c728c4bc8dd20d416d, regionState=CLOSED 2024-12-05T15:49:17,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=83, state=RUNNABLE, hasLock=false; CloseRegionProcedure 003f376bc19761c728c4bc8dd20d416d, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:49:17,091 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/694967a9c2dfe786b3822f9f145f9365/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-12-05T15:49:17,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=83 2024-12-05T15:49:17,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=83, state=SUCCESS, hasLock=false; CloseRegionProcedure 003f376bc19761c728c4bc8dd20d416d, server=5a29627a94df,34815,1733413637643 in 174 msec 2024-12-05T15:49:17,092 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:49:17,092 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:17,092 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 694967a9c2dfe786b3822f9f145f9365: Waiting for close lock at 1733413757084Running coprocessor pre-close hooks at 1733413757084Disabling compacts and flushes for region at 1733413757084Disabling writes for close at 1733413757084Writing region close event to WAL at 1733413757085 (+1 ms)Running coprocessor post-close hooks at 1733413757092 (+7 ms)Closed at 1733413757092 2024-12-05T15:49:17,092 WARN [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: 694967a9c2dfe786b3822f9f145f9365 to self. 2024-12-05T15:49:17,093 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=003f376bc19761c728c4bc8dd20d416d, REOPEN/MOVE; state=CLOSED, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=true 2024-12-05T15:49:17,094 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:17,095 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=694967a9c2dfe786b3822f9f145f9365, regionState=CLOSED 2024-12-05T15:49:17,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 694967a9c2dfe786b3822f9f145f9365, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:49:17,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-05T15:49:17,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 694967a9c2dfe786b3822f9f145f9365, server=5a29627a94df,34815,1733413637643 in 184 msec 2024-12-05T15:49:17,099 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=694967a9c2dfe786b3822f9f145f9365, REOPEN/MOVE; state=CLOSED, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=true 2024-12-05T15:49:17,244 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=694967a9c2dfe786b3822f9f145f9365, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:17,244 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=003f376bc19761c728c4bc8dd20d416d, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:17,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=003f376bc19761c728c4bc8dd20d416d, REOPEN/MOVE because future has completed 2024-12-05T15:49:17,248 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=83, state=RUNNABLE, hasLock=false; OpenRegionProcedure 003f376bc19761c728c4bc8dd20d416d, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:49:17,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=694967a9c2dfe786b3822f9f145f9365, REOPEN/MOVE because future has completed 2024-12-05T15:49:17,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 694967a9c2dfe786b3822f9f145f9365, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:49:17,408 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:17,409 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 003f376bc19761c728c4bc8dd20d416d, NAME => 'table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d.', STARTKEY => '', ENDKEY => 'row0'} 2024-12-05T15:49:17,409 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:49:17,410 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:17,410 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:49:17,410 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:17,410 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:17,413 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:17,414 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 003f376bc19761c728c4bc8dd20d416d columnFamilyName f 2024-12-05T15:49:17,414 DEBUG [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:17,421 DEBUG [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/f/54efbd6b92294e2db01b4d8b1f627d99_SeqId_4_ 2024-12-05T15:49:17,421 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] regionserver.HStore(327): Store=003f376bc19761c728c4bc8dd20d416d/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:17,421 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:17,423 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 003f376bc19761c728c4bc8dd20d416d columnFamilyName f2 2024-12-05T15:49:17,423 DEBUG [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:17,423 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] regionserver.HStore(327): Store=003f376bc19761c728c4bc8dd20d416d/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:17,423 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:17,424 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 003f376bc19761c728c4bc8dd20d416d columnFamilyName mob 2024-12-05T15:49:17,425 DEBUG [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:17,432 DEBUG [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/mob/025555ef841d482f93ea7181a613a078_SeqId_4_ 2024-12-05T15:49:17,432 INFO [StoreOpener-003f376bc19761c728c4bc8dd20d416d-1 {}] regionserver.HStore(327): Store=003f376bc19761c728c4bc8dd20d416d/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:17,433 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:17,434 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:17,435 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:17,436 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:17,436 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:17,437 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T15:49:17,438 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:17,439 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 003f376bc19761c728c4bc8dd20d416d; next sequenceid=7; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10194205920, jitterRate=-0.05059058964252472}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T15:49:17,439 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:17,439 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 003f376bc19761c728c4bc8dd20d416d: Running coprocessor pre-open hook at 1733413757410Writing region info on filesystem at 1733413757411 (+1 ms)Initializing all the Stores at 1733413757412 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413757412Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413757413 (+1 ms)Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413757413Cleaning up temporary data from old regions at 1733413757436 (+23 ms)Running coprocessor post-open hooks at 1733413757439 (+3 ms)Region opened successfully at 1733413757439 2024-12-05T15:49:17,440 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., pid=87, masterSystemTime=1733413757401 2024-12-05T15:49:17,442 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:17,442 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:17,442 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-12-05T15:49:17,442 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:17,442 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => 694967a9c2dfe786b3822f9f145f9365, NAME => 'table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365.', STARTKEY => 'row0', ENDKEY => ''} 2024-12-05T15:49:17,442 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=003f376bc19761c728c4bc8dd20d416d, regionState=OPEN, openSeqNum=7, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:17,443 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:49:17,443 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:17,443 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:49:17,443 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:17,443 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:17,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=83, state=RUNNABLE, hasLock=false; OpenRegionProcedure 003f376bc19761c728c4bc8dd20d416d, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:49:17,445 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:17,446 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 694967a9c2dfe786b3822f9f145f9365 columnFamilyName f 2024-12-05T15:49:17,446 DEBUG [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:17,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=83 2024-12-05T15:49:17,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=83, state=SUCCESS, hasLock=false; OpenRegionProcedure 003f376bc19761c728c4bc8dd20d416d, server=5a29627a94df,34815,1733413637643 in 197 msec 2024-12-05T15:49:17,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=003f376bc19761c728c4bc8dd20d416d, REOPEN/MOVE in 539 msec 2024-12-05T15:49:17,452 DEBUG [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/694967a9c2dfe786b3822f9f145f9365/f/8dded74aa4a74b06b672b010a60967af_SeqId_4_ 2024-12-05T15:49:17,452 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] regionserver.HStore(327): Store=694967a9c2dfe786b3822f9f145f9365/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:17,452 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:17,453 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 694967a9c2dfe786b3822f9f145f9365 columnFamilyName f2 2024-12-05T15:49:17,454 DEBUG [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:17,454 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] regionserver.HStore(327): Store=694967a9c2dfe786b3822f9f145f9365/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:17,454 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:17,455 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 694967a9c2dfe786b3822f9f145f9365 columnFamilyName mob 2024-12-05T15:49:17,455 DEBUG [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:17,456 INFO [StoreOpener-694967a9c2dfe786b3822f9f145f9365-1 {}] regionserver.HStore(327): Store=694967a9c2dfe786b3822f9f145f9365/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:17,456 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:17,457 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:17,458 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:17,458 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:17,459 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:17,459 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T15:49:17,460 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:17,461 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened 694967a9c2dfe786b3822f9f145f9365; next sequenceid=7; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10261660960, jitterRate=-0.04430834949016571}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T15:49:17,461 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:17,461 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for 694967a9c2dfe786b3822f9f145f9365: Running coprocessor pre-open hook at 1733413757443Writing region info on filesystem at 1733413757443Initializing all the Stores at 1733413757444 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413757444Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413757445 (+1 ms)Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413757445Cleaning up temporary data from old regions at 1733413757459 (+14 ms)Running coprocessor post-open hooks at 1733413757461 (+2 ms)Region opened successfully at 1733413757461 2024-12-05T15:49:17,462 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365., pid=88, masterSystemTime=1733413757401 2024-12-05T15:49:17,464 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:17,464 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:17,464 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=694967a9c2dfe786b3822f9f145f9365, regionState=OPEN, openSeqNum=7, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:17,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 694967a9c2dfe786b3822f9f145f9365, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:49:17,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=84 2024-12-05T15:49:17,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 694967a9c2dfe786b3822f9f145f9365, server=5a29627a94df,34815,1733413637643 in 217 msec 2024-12-05T15:49:17,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=82 2024-12-05T15:49:17,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=82, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=694967a9c2dfe786b3822f9f145f9365, REOPEN/MOVE in 561 msec 2024-12-05T15:49:17,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-05T15:49:17,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=table1 in 567 msec 2024-12-05T15:49:17,475 DEBUG [PEWorker-3 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=table1 2024-12-05T15:49:17,476 DEBUG [PEWorker-3 {}] backup.HFileArchiver(265): No files to dispose of in table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., family=f3 2024-12-05T15:49:17,476 DEBUG [PEWorker-3 {}] backup.HFileArchiver(265): No files to dispose of in table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365., family=f3 2024-12-05T15:49:17,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; ModifyTableProcedure table=table1 in 996 msec 2024-12-05T15:49:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-05T15:49:17,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:table1 completed 2024-12-05T15:49:17,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-05T15:49:17,484 INFO [Time-limited test {}] util.RestoreTool(200): Changed table1 to: 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:49:17,484 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore incremental backup from directory hdfs://localhost:37233/backupUT/backup_1733413727346/default/test-1733413646382/f2/f294e4172be34af5990ccf7377ca6be0 from hbase tables test-1733413646382 to tables table1 2024-12-05T15:49:17,484 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore test-1733413646382 into table1 2024-12-05T15:49:17,485 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413757484 from hdfs://localhost:37233/backupUT/backup_1733413727346/default/test-1733413646382/f2/f294e4172be34af5990ccf7377ca6be0 2024-12-05T15:49:17,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47590500, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:17,486 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:17,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:17,487 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:17,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:17,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:17,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32aca88a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:17,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:17,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:17,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:17,488 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:17,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197faf93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:17,489 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:17,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:17,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1a003455 2024-12-05T15:49:17,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:17,491 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:17,492 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-05T15:49:17,492 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table table1 2024-12-05T15:49:17,492 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-05T15:49:17,492 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:17,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:17,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:17,494 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:17,496 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: 2024-12-05T15:49:17,496 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: row0 2024-12-05T15:49:17,496 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-12-05T15:49:17,496 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/partitions_a69c0d2b-9cad-4984-91ce-3962abd8ac76 2024-12-05T15:49:17,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741990_1166 (size=177) 2024-12-05T15:49:17,905 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:17,906 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:17,907 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,667 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-6707878230997400999.jar 2024-12-05T15:49:18,668 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,668 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,668 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,668 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,668 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,669 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,669 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,669 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,669 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,669 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:49:18,670 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-05T15:49:18,670 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-05T15:49:18,670 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-05T15:49:18,670 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-05T15:49:18,670 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-05T15:49:18,671 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-05T15:49:18,671 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-05T15:49:18,671 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-05T15:49:18,671 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-05T15:49:18,671 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-05T15:49:18,671 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,672 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,672 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,672 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,672 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,672 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:18,672 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:49:18,673 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: table1 2024-12-05T15:49:18,673 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:49:18,673 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.incrementalRestoreTable(RestoreTool.java:205) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:185) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:218) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:18,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:18,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:18,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:18,673 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-12-05T15:49:18,673 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:49:18,683 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:49:18,694 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-12-05T15:49:18,772 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local2113490872_0015_c859b9c5-1e56-4566-b3f5-2aab491bfe67/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-05T15:49:18,772 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local2113490872_0015_c859b9c5-1e56-4566-b3f5-2aab491bfe67/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-05T15:49:18,824 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:37233/backupUT/backup_1733413727346/default/test-1733413646382/f2/f294e4172be34af5990ccf7377ca6be0 2024-12-05T15:49:18,827 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-05T15:49:18,857 DEBUG [pool-281-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c902a3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:18,857 DEBUG [pool-281-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:18,857 DEBUG [pool-281-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:18,858 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:18,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:18,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:18,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ade0ea0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:18,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:18,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:18,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:18,860 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:18,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bb13ebc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:18,861 DEBUG [pool-281-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:18,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:18,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:18,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:18,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1733413646382-2-0', locateType=CURRENT is [region=table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., hostname=5a29627a94df,34815,1733413637643, seqNum=7] 2024-12-05T15:49:18,865 INFO [pool-281-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-281-thread-1. 2024-12-05T15:49:18,865 DEBUG [pool-281-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:18,865 DEBUG [pool-281-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:18,865 DEBUG [pool-281-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:18,865 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:18,865 DEBUG [pool-281-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733413646382-2-0] 2024-12-05T15:49:18,865 DEBUG [pool-281-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5a29627a94df 2024-12-05T15:49:18,869 DEBUG [pool-281-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413757484/_temporary/0/_temporary/attempt_local2113490872_0015_r_000000_0/f2/d718a83a15614cffa8c4fdc45a41e637 is 58, key is row-test-1733413646382-2-0/f2:q1/1733413722337/Put/seqid=0 2024-12-05T15:49:18,871 WARN [Thread-1887 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5a29627a94df:34815] Specified favored nodes: [5a29627a94df:34815] 2024-12-05T15:49:18,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741991_1167 (size=5609) 2024-12-05T15:49:19,806 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413757484 2024-12-05T15:49:19,806 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a6aaf86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:19,806 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:19,806 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:19,807 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:19,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:19,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:19,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123ef76d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:19,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:19,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:19,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:19,809 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:19,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17d4626d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:19,810 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:19,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:19,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:19,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:19,815 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413757484/_SUCCESS 2024-12-05T15:49:19,818 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:19,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:19,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c09b3fd 2024-12-05T15:49:19,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:19,821 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:19,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., hostname=5a29627a94df,34815,1733413637643, seqNum=7] 2024-12-05T15:49:19,826 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-12-05T15:49:19,832 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413757484/f2/d718a83a15614cffa8c4fdc45a41e637 first=Optional[row-test-1733413646382-2-0] last=Optional[row-test-1733413646382-2-6] 2024-12-05T15:49:19,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413757484/f2/d718a83a15614cffa8c4fdc45a41e637 for inclusion in 003f376bc19761c728c4bc8dd20d416d/f2 2024-12-05T15:49:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(626): HFile bounds: first=row-test-1733413646382-2-0 last=row-test-1733413646382-2-6 2024-12-05T15:49:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-12-05T15:49:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HRegion(2603): Flush status journal for 003f376bc19761c728c4bc8dd20d416d: 2024-12-05T15:49:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-default-table1-1733413757484/f2/d718a83a15614cffa8c4fdc45a41e637 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__m7hhuirdqnhv70cb85vqllhkpvrn3k2ohmce7hu2jdf2g8g9s1k1k3jsgnbb6kjm/f2/d718a83a15614cffa8c4fdc45a41e637 2024-12-05T15:49:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__m7hhuirdqnhv70cb85vqllhkpvrn3k2ohmce7hu2jdf2g8g9s1k1k3jsgnbb6kjm/f2/d718a83a15614cffa8c4fdc45a41e637 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/f2/faa00360a8ea4b30882ef35a4ff958d5_SeqId_9_ 2024-12-05T15:49:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57f52b52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:19,844 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:19,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:19,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:19,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aa588a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:19,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:19,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:19,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:19,845 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:19,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dcda893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:19,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:19,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:19,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4cb153fd 2024-12-05T15:49:19,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:19,847 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:19,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:19,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:19,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:19,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:19,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:19,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:19,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815. 2024-12-05T15:49:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:19,863 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:19,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__m7hhuirdqnhv70cb85vqllhkpvrn3k2ohmce7hu2jdf2g8g9s1k1k3jsgnbb6kjm/f2/d718a83a15614cffa8c4fdc45a41e637 into 003f376bc19761c728c4bc8dd20d416d/f2 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/f2/faa00360a8ea4b30882ef35a4ff958d5_SeqId_9_ - updating store file list. 2024-12-05T15:49:19,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/f2/faa00360a8ea4b30882ef35a4ff958d5_SeqId_9_ into 003f376bc19761c728c4bc8dd20d416d/f2 2024-12-05T15:49:19,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__m7hhuirdqnhv70cb85vqllhkpvrn3k2ohmce7hu2jdf2g8g9s1k1k3jsgnbb6kjm/f2/d718a83a15614cffa8c4fdc45a41e637 into 003f376bc19761c728c4bc8dd20d416d/f2 (new location: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/f2/faa00360a8ea4b30882ef35a4ff958d5_SeqId_9_) 2024-12-05T15:49:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__table1__m7hhuirdqnhv70cb85vqllhkpvrn3k2ohmce7hu2jdf2g8g9s1k1k3jsgnbb6kjm/f2/d718a83a15614cffa8c4fdc45a41e637 2024-12-05T15:49:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd09fe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:19,871 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:19,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:19,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:19,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7a80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:19,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:19,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:19,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:19,872 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:19,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c902e01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:19,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:19,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2ab61d77 2024-12-05T15:49:19,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:19,874 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-05T15:49:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:19,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:19,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:19,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-05T15:49:19,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:19,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:19,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:19,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815. 2024-12-05T15:49:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:19,889 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:19,890 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:49:19,890 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.incrementalRestoreTable(RestoreTool.java:205) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:185) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:218) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:19,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:19,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:19,890 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:19,890 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-12-05T15:49:19,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., hostname=5a29627a94df,34815,1733413637643, seqNum=7 , the old value is region=table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., hostname=5a29627a94df,34815,1733413637643, seqNum=7, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=5a29627a94df:34815 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-05T15:49:19,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., hostname=5a29627a94df,34815,1733413637643, seqNum=7 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-05T15:49:19,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., hostname=5a29627a94df,34815,1733413637643, seqNum=7 from cache 2024-12-05T15:49:19,891 INFO [Time-limited test {}] impl.RestoreTablesClient(187): test-1733413646382 has been successfully restored to table1 2024-12-05T15:49:19,891 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-12-05T15:49:19,891 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1733413727346 hdfs://localhost:37233/backupUT/backup_1733413727346/default/test-1733413646382/ 2024-12-05T15:49:19,891 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-05T15:49:19,892 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:37233/backupUT/backup_1733413692606/.backup.manifest 2024-12-05T15:49:19,893 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733413692606/.backup.manifest 2024-12-05T15:49:19,893 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'ns2:test-17334136463821' to 'ns2:table2' from full backup image hdfs://localhost:37233/backupUT/backup_1733413692606/ns2/test-17334136463821 2024-12-05T15:49:19,895 DEBUG [Time-limited test {}] util.RestoreTool(295): Retrieved descriptor: 'ns2:test-17334136463821', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} thru backup_1733413727346 2024-12-05T15:49:19,900 INFO [Time-limited test {}] util.RestoreTool(480): Truncating exising target table 'ns2:table2', preserving region splits 2024-12-05T15:49:19,900 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable ns2:table2 2024-12-05T15:49:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=ns2:table2 2024-12-05T15:49:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T15:49:19,904 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413759904"}]},"ts":"1733413759904"} 2024-12-05T15:49:19,906 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2024-12-05T15:49:19,906 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set ns2:table2 to state=DISABLING 2024-12-05T15:49:19,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=ns2:table2}] 2024-12-05T15:49:19,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=4c2f6298605fad0624d0dec8fedda830, UNASSIGN}] 2024-12-05T15:49:19,910 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=4c2f6298605fad0624d0dec8fedda830, UNASSIGN 2024-12-05T15:49:19,910 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=91 updating hbase:meta row=4c2f6298605fad0624d0dec8fedda830, regionState=CLOSING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:19,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=4c2f6298605fad0624d0dec8fedda830, UNASSIGN because future has completed 2024-12-05T15:49:19,912 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T15:49:19,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c2f6298605fad0624d0dec8fedda830, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:49:19,997 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'table1', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=5a29627a94df:34815 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 34 more 2024-12-05T15:49:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T15:49:20,065 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] handler.UnassignRegionHandler(122): Close 4c2f6298605fad0624d0dec8fedda830 2024-12-05T15:49:20,065 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T15:49:20,065 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1722): Closing 4c2f6298605fad0624d0dec8fedda830, disabling compactions & flushes 2024-12-05T15:49:20,065 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1755): Closing region ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. 2024-12-05T15:49:20,065 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. 2024-12-05T15:49:20,066 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. after waiting 0 ms 2024-12-05T15:49:20,066 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. 2024-12-05T15:49:20,075 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-12-05T15:49:20,076 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:49:20,076 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1973): Closed ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. 2024-12-05T15:49:20,076 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1676): Region close journal for 4c2f6298605fad0624d0dec8fedda830: Waiting for close lock at 1733413760065Running coprocessor pre-close hooks at 1733413760065Disabling compacts and flushes for region at 1733413760065Disabling writes for close at 1733413760066 (+1 ms)Writing region close event to WAL at 1733413760067 (+1 ms)Running coprocessor post-close hooks at 1733413760076 (+9 ms)Closed at 1733413760076 2024-12-05T15:49:20,080 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] handler.UnassignRegionHandler(157): Closed 4c2f6298605fad0624d0dec8fedda830 2024-12-05T15:49:20,081 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=91 updating hbase:meta row=4c2f6298605fad0624d0dec8fedda830, regionState=CLOSED 2024-12-05T15:49:20,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c2f6298605fad0624d0dec8fedda830, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:49:20,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-05T15:49:20,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c2f6298605fad0624d0dec8fedda830, server=5a29627a94df,34815,1733413637643 in 173 msec 2024-12-05T15:49:20,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-05T15:49:20,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=4c2f6298605fad0624d0dec8fedda830, UNASSIGN in 179 msec 2024-12-05T15:49:20,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-05T15:49:20,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=ns2:table2 in 185 msec 2024-12-05T15:49:20,097 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413760097"}]},"ts":"1733413760097"} 2024-12-05T15:49:20,099 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2024-12-05T15:49:20,100 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set ns2:table2 to state=DISABLED 2024-12-05T15:49:20,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; DisableTableProcedure table=ns2:table2 in 201 msec 2024-12-05T15:49:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T15:49:20,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: ns2:table2 completed 2024-12-05T15:49:20,220 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] master.HMaster$6(2602): Client=jenkins//172.17.0.3 truncate ns2:table2 2024-12-05T15:49:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION, hasLock=false; TruncateTableProcedure (table=ns2:table2 preserveSplits=true) 2024-12-05T15:49:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T15:49:20,224 DEBUG [PEWorker-5 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:table2' regions in transition 2024-12-05T15:49:20,229 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830 2024-12-05T15:49:20,231 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830/f, FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830/recovered.edits] 2024-12-05T15:49:20,238 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830/f/28db3e03550e42c8b8390f8ae5f5887f_SeqId_4_ to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/archive/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830/f/28db3e03550e42c8b8390f8ae5f5887f_SeqId_4_ 2024-12-05T15:49:20,243 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830/recovered.edits/6.seqid to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/archive/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830/recovered.edits/6.seqid 2024-12-05T15:49:20,244 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/4c2f6298605fad0624d0dec8fedda830 2024-12-05T15:49:20,244 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived ns2:table2 regions 2024-12-05T15:49:20,251 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2024-12-05T15:49:20,253 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'ns2:table2' descriptor. 2024-12-05T15:49:20,254 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'ns2:table2' from region states. 2024-12-05T15:49:20,254 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733413760254"}]},"ts":"9223372036854775807"} 2024-12-05T15:49:20,256 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T15:49:20,256 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4c2f6298605fad0624d0dec8fedda830, NAME => 'ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T15:49:20,256 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'ns2:table2' as deleted. 2024-12-05T15:49:20,256 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733413760256"}]},"ts":"9223372036854775807"} 2024-12-05T15:49:20,258 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table ns2:table2 state from META 2024-12-05T15:49:20,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741992_1168 (size=351) 2024-12-05T15:49:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T15:49:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T15:49:20,669 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 94f5b136d6cdc9f73ab4b3495ebd0d3a, NAME => 'ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089 2024-12-05T15:49:20,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741993_1169 (size=37) 2024-12-05T15:49:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T15:49:20,987 DEBUG [regionserver/5a29627a94df:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-12-05T15:49:21,081 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:49:21,081 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1722): Closing 94f5b136d6cdc9f73ab4b3495ebd0d3a, disabling compactions & flushes 2024-12-05T15:49:21,081 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a. 2024-12-05T15:49:21,081 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a. 2024-12-05T15:49:21,082 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a. after waiting 0 ms 2024-12-05T15:49:21,082 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a. 2024-12-05T15:49:21,082 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1973): Closed ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a. 2024-12-05T15:49:21,082 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1676): Region close journal for 94f5b136d6cdc9f73ab4b3495ebd0d3a: Waiting for close lock at 1733413761081Disabling compacts and flushes for region at 1733413761081Disabling writes for close at 1733413761082 (+1 ms)Writing region close event to WAL at 1733413761082Closed at 1733413761082 2024-12-05T15:49:21,084 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1733413761083"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733413761083"}]},"ts":"1733413761083"} 2024-12-05T15:49:21,087 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T15:49:21,088 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413761088"}]},"ts":"1733413761088"} 2024-12-05T15:49:21,091 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2024-12-05T15:49:21,091 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=94f5b136d6cdc9f73ab4b3495ebd0d3a, ASSIGN}] 2024-12-05T15:49:21,093 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=94f5b136d6cdc9f73ab4b3495ebd0d3a, ASSIGN 2024-12-05T15:49:21,093 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=94f5b136d6cdc9f73ab4b3495ebd0d3a, ASSIGN; state=OFFLINE, location=5a29627a94df,34815,1733413637643; forceNewPlan=false, retain=false 2024-12-05T15:49:21,245 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=94f5b136d6cdc9f73ab4b3495ebd0d3a, regionState=OPENING, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:21,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=94f5b136d6cdc9f73ab4b3495ebd0d3a, ASSIGN because future has completed 2024-12-05T15:49:21,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94f5b136d6cdc9f73ab4b3495ebd0d3a, server=5a29627a94df,34815,1733413637643}] 2024-12-05T15:49:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T15:49:21,413 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a. 2024-12-05T15:49:21,413 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 94f5b136d6cdc9f73ab4b3495ebd0d3a, NAME => 'ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a.', STARTKEY => '', ENDKEY => ''} 2024-12-05T15:49:21,414 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-05T15:49:21,414 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 94f5b136d6cdc9f73ab4b3495ebd0d3a 2024-12-05T15:49:21,414 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T15:49:21,415 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 94f5b136d6cdc9f73ab4b3495ebd0d3a 2024-12-05T15:49:21,415 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 94f5b136d6cdc9f73ab4b3495ebd0d3a 2024-12-05T15:49:21,419 INFO [StoreOpener-94f5b136d6cdc9f73ab4b3495ebd0d3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 94f5b136d6cdc9f73ab4b3495ebd0d3a 2024-12-05T15:49:21,420 INFO [StoreOpener-94f5b136d6cdc9f73ab4b3495ebd0d3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94f5b136d6cdc9f73ab4b3495ebd0d3a columnFamilyName f 2024-12-05T15:49:21,421 DEBUG [StoreOpener-94f5b136d6cdc9f73ab4b3495ebd0d3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:21,421 INFO [StoreOpener-94f5b136d6cdc9f73ab4b3495ebd0d3a-1 {}] regionserver.HStore(327): Store=94f5b136d6cdc9f73ab4b3495ebd0d3a/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T15:49:21,421 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 94f5b136d6cdc9f73ab4b3495ebd0d3a 2024-12-05T15:49:21,423 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/94f5b136d6cdc9f73ab4b3495ebd0d3a 2024-12-05T15:49:21,423 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/94f5b136d6cdc9f73ab4b3495ebd0d3a 2024-12-05T15:49:21,424 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 94f5b136d6cdc9f73ab4b3495ebd0d3a 2024-12-05T15:49:21,424 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 94f5b136d6cdc9f73ab4b3495ebd0d3a 2024-12-05T15:49:21,427 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 94f5b136d6cdc9f73ab4b3495ebd0d3a 2024-12-05T15:49:21,430 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/94f5b136d6cdc9f73ab4b3495ebd0d3a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T15:49:21,430 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 94f5b136d6cdc9f73ab4b3495ebd0d3a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9857483200, jitterRate=-0.08195033669471741}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T15:49:21,430 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94f5b136d6cdc9f73ab4b3495ebd0d3a 2024-12-05T15:49:21,431 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 94f5b136d6cdc9f73ab4b3495ebd0d3a: Running coprocessor pre-open hook at 1733413761415Writing region info on filesystem at 1733413761415Initializing all the Stores at 1733413761417 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733413761418 (+1 ms)Cleaning up temporary data from old regions at 1733413761424 (+6 ms)Running coprocessor post-open hooks at 1733413761430 (+6 ms)Region opened successfully at 1733413761431 (+1 ms) 2024-12-05T15:49:21,431 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a., pid=95, masterSystemTime=1733413761405 2024-12-05T15:49:21,433 DEBUG [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a. 2024-12-05T15:49:21,433 INFO [RS_OPEN_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a. 2024-12-05T15:49:21,434 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=94f5b136d6cdc9f73ab4b3495ebd0d3a, regionState=OPEN, openSeqNum=2, regionLocation=5a29627a94df,34815,1733413637643 2024-12-05T15:49:21,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94f5b136d6cdc9f73ab4b3495ebd0d3a, server=5a29627a94df,34815,1733413637643 because future has completed 2024-12-05T15:49:21,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-05T15:49:21,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 94f5b136d6cdc9f73ab4b3495ebd0d3a, server=5a29627a94df,34815,1733413637643 in 185 msec 2024-12-05T15:49:21,439 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-05T15:49:21,439 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=94f5b136d6cdc9f73ab4b3495ebd0d3a, ASSIGN in 346 msec 2024-12-05T15:49:21,439 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733413761439"}]},"ts":"1733413761439"} 2024-12-05T15:49:21,440 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2024-12-05T15:49:21,440 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:table2' completed 2024-12-05T15:49:21,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; TruncateTableProcedure (table=ns2:table2 preserveSplits=true) in 1.2200 sec 2024-12-05T15:49:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33117 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T15:49:22,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: TRUNCATE, Table Name: ns2:table2 completed 2024-12-05T15:49:22,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-12-05T15:49:22,377 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:37233/backupUT/backup_1733413692606/ns2/test-17334136463821/archive/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728 from hbase tables ns2:test-17334136463821 to tables ns2:table2 2024-12-05T15:49:22,378 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-17334136463821 into ns2:table2 2024-12-05T15:49:22,380 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-ns2-table2-1733413762378 from hdfs://localhost:37233/backupUT/backup_1733413692606/ns2/test-17334136463821/archive/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728 2024-12-05T15:49:22,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70531e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:22,381 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:22,382 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:22,383 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:22,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:22,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:22,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@492d2d44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:22,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:22,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:22,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:22,385 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:22,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1270b950, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:22,387 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:22,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:22,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@355d84b7 2024-12-05T15:49:22,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:22,389 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:22,389 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-05T15:49:22,390 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:table2 2024-12-05T15:49:22,390 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-12-05T15:49:22,390 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:22,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:22,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:22,392 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:22,394 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:table2: 2024-12-05T15:49:22,394 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 1 reduce partitions to match current region count for all tables 2024-12-05T15:49:22,394 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/partitions_f2101231-6fe0-4adb-9a4d-64dc076baa86 2024-12-05T15:49:22,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741994_1170 (size=153) 2024-12-05T15:49:22,585 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-12-05T15:49:22,803 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:22,803 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:22,804 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,566 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop-5159527294152622081.jar 2024-12-05T15:49:23,566 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,567 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,567 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,567 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,567 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,567 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,568 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,568 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,568 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,568 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:49:23,568 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-05T15:49:23,569 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-05T15:49:23,569 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-05T15:49:23,569 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-05T15:49:23,569 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-05T15:49:23,569 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-05T15:49:23,569 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-05T15:49:23,570 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-05T15:49:23,570 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-05T15:49:23,570 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-05T15:49:23,570 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,570 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,570 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,571 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,571 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,571 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-05T15:49:23,571 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-05T15:49:23,571 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:table2 2024-12-05T15:49:23,571 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:49:23,571 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:218) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:23,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:23,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:23,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:23,572 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-12-05T15:49:23,572 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-05T15:49:23,581 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-05T15:49:23,597 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-12-05T15:49:23,675 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local2084756555_0016_f3a2201b-e6c1-4c0c-8e67-efd479dd3179/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-05T15:49:23,675 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/mapreduce.cluster.local.dir/job_local2084756555_0016_f3a2201b-e6c1-4c0c-8e67-efd479dd3179/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-05T15:49:23,717 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:37233/backupUT/backup_1733413692606/ns2/test-17334136463821/archive/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/1159a028de3844c4b30999226eb32483 2024-12-05T15:49:23,721 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-05T15:49:23,751 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:37233/backupUT/backup_1733413692606/ns2/test-17334136463821/archive/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/f/d97c997a6e4d4d18b6b69960413bfcdd 2024-12-05T15:49:23,755 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-12-05T15:49:23,784 DEBUG [pool-286-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d33e031, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:23,785 DEBUG [pool-286-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:23,785 DEBUG [pool-286-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:23,786 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:23,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:23,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:23,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a051db1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:23,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:23,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:23,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:23,787 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:23,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f97c30b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:23,788 DEBUG [pool-286-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:23,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:23,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:23,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:23,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='row-t20', locateType=CURRENT is [region=ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:23,792 INFO [pool-286-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-286-thread-1. 2024-12-05T15:49:23,792 DEBUG [pool-286-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:23,792 DEBUG [pool-286-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:23,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:23,792 DEBUG [pool-286-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:23,792 DEBUG [pool-286-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-12-05T15:49:23,792 DEBUG [pool-286-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5a29627a94df 2024-12-05T15:49:23,796 DEBUG [pool-286-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-ns2-table2-1733413762378/_temporary/0/_temporary/attempt_local2084756555_0016_r_000000_0/f/9364d1ca28b74099925f84508f7f5060 is 38, key is row-t20/f:q1/1733413677638/Put/seqid=0 2024-12-05T15:49:23,798 WARN [Thread-1962 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5a29627a94df:34815] Specified favored nodes: [5a29627a94df:34815] 2024-12-05T15:49:23,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741995_1171 (size=8935) 2024-12-05T15:49:24,716 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-ns2-table2-1733413762378 2024-12-05T15:49:24,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e658e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:24,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:24,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:24,718 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:24,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:24,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:24,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2633190b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:24,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:24,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:24,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:24,720 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:24,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43dda463, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:24,721 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:24,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:24,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:24,722 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:24,725 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-ns2-table2-1733413762378/_SUCCESS 2024-12-05T15:49:24,728 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:24,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:24,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3f20185d 2024-12-05T15:49:24,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:24,729 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:24,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:24,734 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-12-05T15:49:24,739 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-ns2-table2-1733413762378/f/9364d1ca28b74099925f84508f7f5060 first=Optional[row-t20] last=Optional[row98] 2024-12-05T15:49:24,743 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-ns2-table2-1733413762378/f/9364d1ca28b74099925f84508f7f5060 for inclusion in 94f5b136d6cdc9f73ab4b3495ebd0d3a/f 2024-12-05T15:49:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(626): HFile bounds: first=row-t20 last=row98 2024-12-05T15:49:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-05T15:49:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HRegion(2603): Flush status journal for 94f5b136d6cdc9f73ab4b3495ebd0d3a: 2024-12-05T15:49:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/hbase-staging/bulk_output-ns2-table2-1733413762378/f/9364d1ca28b74099925f84508f7f5060 to hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__ns2_table2__8srrddk9nnrvarpehao1v6jlkhc4ngtmegj63i9d6ee206e982q8hs3370ep6199/f/9364d1ca28b74099925f84508f7f5060 2024-12-05T15:49:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__ns2_table2__8srrddk9nnrvarpehao1v6jlkhc4ngtmegj63i9d6ee206e982q8hs3370ep6199/f/9364d1ca28b74099925f84508f7f5060 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/94f5b136d6cdc9f73ab4b3495ebd0d3a/f/a48cf95a2efc4f11895c49e3cc40a858_SeqId_4_ 2024-12-05T15:49:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1350f53c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:24,752 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:24,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:24,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:24,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5580aa0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:24,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:24,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:24,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:24,754 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:24,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@783d5332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:24,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:24,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@26c20489 2024-12-05T15:49:24,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:24,757 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T15:49:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:24,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:24,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:24,759 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T15:49:24,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:24,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:24,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:24,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815. 2024-12-05T15:49:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:24,773 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:24,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__ns2_table2__8srrddk9nnrvarpehao1v6jlkhc4ngtmegj63i9d6ee206e982q8hs3370ep6199/f/9364d1ca28b74099925f84508f7f5060 into 94f5b136d6cdc9f73ab4b3495ebd0d3a/f as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/94f5b136d6cdc9f73ab4b3495ebd0d3a/f/a48cf95a2efc4f11895c49e3cc40a858_SeqId_4_ - updating store file list. 2024-12-05T15:49:24,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/94f5b136d6cdc9f73ab4b3495ebd0d3a/f/a48cf95a2efc4f11895c49e3cc40a858_SeqId_4_ into 94f5b136d6cdc9f73ab4b3495ebd0d3a/f 2024-12-05T15:49:24,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__ns2_table2__8srrddk9nnrvarpehao1v6jlkhc4ngtmegj63i9d6ee206e982q8hs3370ep6199/f/9364d1ca28b74099925f84508f7f5060 into 94f5b136d6cdc9f73ab4b3495ebd0d3a/f (new location: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/94f5b136d6cdc9f73ab4b3495ebd0d3a/f/a48cf95a2efc4f11895c49e3cc40a858_SeqId_4_) 2024-12-05T15:49:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/staging/jenkins__ns2_table2__8srrddk9nnrvarpehao1v6jlkhc4ngtmegj63i9d6ee206e982q8hs3370ep6199/f/9364d1ca28b74099925f84508f7f5060 2024-12-05T15:49:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60e07737, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ClusterIdFetcher(90): Going to request 5a29627a94df,33117,-1 for getting cluster id 2024-12-05T15:49:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T15:49:24,781 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ce2a395-fcd1-42e7-9dc1-a423aee17fc6' 2024-12-05T15:49:24,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T15:49:24,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ce2a395-fcd1-42e7-9dc1-a423aee17fc6" 2024-12-05T15:49:24,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d7bb846, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:24,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5a29627a94df,33117,-1] 2024-12-05T15:49:24,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T15:49:24,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:24,782 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-12-05T15:49:24,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e77bd0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-05T15:49:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T15:49:24,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 5a29627a94df,33117,1733413636177 2024-12-05T15:49:24,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@46147965 2024-12-05T15:49:24,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T15:49:24,784 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-05T15:49:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T15:49:24,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5a29627a94df,34815,1733413637643, seqNum=-1] 2024-12-05T15:49:24,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T15:49:24,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-05T15:49:24,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-05T15:49:24,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-05T15:49:24,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:24,799 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815. 2024-12-05T15:49:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:24,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:24,800 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:49:24,800 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:218) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:24,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:24,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:24,800 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:24,800 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-12-05T15:49:24,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a., hostname=5a29627a94df,34815,1733413637643, seqNum=2 , the old value is region=ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a., hostname=5a29627a94df,34815,1733413637643, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=5a29627a94df:34815 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-05T15:49:24,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a., hostname=5a29627a94df,34815,1733413637643, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-05T15:49:24,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a., hostname=5a29627a94df,34815,1733413637643, seqNum=2 from cache 2024-12-05T15:49:24,801 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-05T15:49:24,815 INFO [Time-limited test {}] impl.RestoreTablesClient(175): No incremental changes since full backup for 'ns2:test-17334136463821', skipping incremental restore step. 2024-12-05T15:49:24,815 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-12-05T15:49:24,815 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1733413727346 hdfs://localhost:37233/backupUT/backup_1733413727346/ns2/test-17334136463821/ 2024-12-05T15:49:24,815 DEBUG [Time-limited test {}] impl.RestoreTablesClient(236): restoreStage finished 2024-12-05T15:49:24,816 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(222): After incremental restore: 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T15:49:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.CallRunner(138): callId: 742 service: ClientService methodName: Scan size: 160 connection: 172.17.0.3:40976 deadline: 1733413824816, exception=org.apache.hadoop.hbase.NotServingRegionException: table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. is not online on 5a29627a94df,34815,1733413637643 2024-12-05T15:49:24,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3., hostname=5a29627a94df,34815,1733413637643, seqNum=2 , the old value is region=table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3., hostname=5a29627a94df,34815,1733413637643, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. is not online on 5a29627a94df,34815,1733413637643 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:24,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3., hostname=5a29627a94df,34815,1733413637643, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3. is not online on 5a29627a94df,34815,1733413637643 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:24,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1733413740066.98789158a6f27fe53f9525b6fc318ac3., hostname=5a29627a94df,34815,1733413637643, seqNum=2 from cache 2024-12-05T15:49:24,907 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'ns2:table2', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=5a29627a94df:34815 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 34 more 2024-12-05T15:49:24,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d., hostname=5a29627a94df,34815,1733413637643, seqNum=7] 2024-12-05T15:49:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.CallRunner(138): callId: 746 service: ClientService methodName: Scan size: 170 connection: 172.17.0.3:40976 deadline: 1733413824935, exception=org.apache.hadoop.hbase.NotServingRegionException: table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. is not online on 5a29627a94df,34815,1733413637643 2024-12-05T15:49:24,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3., hostname=5a29627a94df,34815,1733413637643, seqNum=2 , the old value is region=table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3., hostname=5a29627a94df,34815,1733413637643, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. is not online on 5a29627a94df,34815,1733413637643 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:24,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3., hostname=5a29627a94df,34815,1733413637643, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3. is not online on 5a29627a94df,34815,1733413637643 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:24,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,row0,1733413740066.8449bae89aa1e699ba056d7b9b3032d3., hostname=5a29627a94df,34815,1733413637643, seqNum=2 from cache 2024-12-05T15:49:25,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365., hostname=5a29627a94df,34815,1733413637643, seqNum=7] 2024-12-05T15:49:25,058 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(224): f1 has 198 rows 2024-12-05T15:49:25,064 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(228): f2 has 7 rows 2024-12-05T15:49:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:25,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-12-05T15:49:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:25,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-12-05T15:49:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:25,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-12-05T15:49:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:25,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-12-05T15:49:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:25,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-12-05T15:49:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T15:49:25,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-12-05T15:49:25,088 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(232): mob has 111 rows 2024-12-05T15:49:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34815 {}] ipc.CallRunner(138): callId: 754 service: ClientService methodName: Scan size: 159 connection: 172.17.0.3:40976 deadline: 1733413825088, exception=org.apache.hadoop.hbase.NotServingRegionException: ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. is not online on 5a29627a94df,34815,1733413637643 2024-12-05T15:49:25,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830., hostname=5a29627a94df,34815,1733413637643, seqNum=2 , the old value is region=ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830., hostname=5a29627a94df,34815,1733413637643, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. is not online on 5a29627a94df,34815,1733413637643 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:25,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830., hostname=5a29627a94df,34815,1733413637643, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830. is not online on 5a29627a94df,34815,1733413637643 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T15:49:25,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830., hostname=5a29627a94df,34815,1733413637643, seqNum=2 from cache 2024-12-05T15:49:25,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a., hostname=5a29627a94df,34815,1733413637643, seqNum=2] 2024-12-05T15:49:25,206 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:49:25,206 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:240) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:25,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:25,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:25,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:25,220 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=635 (was 610) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-796844295_20 at /127.0.0.1:43888 [Receiving block BP-671471983-172.17.0.3-1733413632680:blk_1073741955_1131] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-671471983-172.17.0.3-1733413632680:blk_1073741956_1132, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-796844295_20 at /127.0.0.1:43880 [Receiving block BP-671471983-172.17.0.3-1733413632680:blk_1073741954_1130] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/cluster_783e9d64-ce48-dc27-a242-83d4c51d286c/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: (5a29627a94df,33117,1733413636177)-proc-coordinator-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(5a29627a94df,34815,1733413637643)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: member: '5a29627a94df,34815,1733413637643' subprocedure-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(5a29627a94df,34815,1733413637643)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/5a29627a94df:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(5a29627a94df,34815,1733413637643)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'JobTracker' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-671471983-172.17.0.3-1733413632680:blk_1073741954_1130, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/5a29627a94df:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-796844295_20 at /127.0.0.1:43892 [Receiving block BP-671471983-172.17.0.3-1733413632680:blk_1073741956_1132] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(5a29627a94df,34815,1733413637643)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1805364529_20 at /127.0.0.1:55912 [Waiting for operation #54] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/cluster_783e9d64-ce48-dc27-a242-83d4c51d286c/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-671471983-172.17.0.3-1733413632680:blk_1073741955_1131, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=643 (was 633) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=101 (was 126), ProcessCount=18 (was 18), AvailableMemoryMB=8799 (was 9338) 2024-12-05T15:49:25,220 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=635 is superior to 500 2024-12-05T15:49:25,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T15:49:25,227 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:49:25,227 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:25,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:25,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:25,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:25,227 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T15:49:25,227 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=391921891, stopped=false 2024-12-05T15:49:25,228 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=5a29627a94df,33117,1733413636177 2024-12-05T15:49:25,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-12-05T15:49:25,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-12-05T15:49:25,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:49:25,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:49:25,272 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T15:49:25,273 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T15:49:25,273 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:25,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:25,274 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-05T15:49:25,274 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5a29627a94df,34815,1733413637643' ***** 2024-12-05T15:49:25,274 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T15:49:25,274 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-05T15:49:25,275 INFO [RS:0;5a29627a94df:34815 {}] hbase.HBaseServerBase(468): Stop info server 2024-12-05T15:49:25,282 INFO [RS:0;5a29627a94df:34815 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@6edf2259{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-12-05T15:49:25,284 INFO [RS:0;5a29627a94df:34815 {}] server.AbstractConnector(383): Stopped ServerConnector@1aa5419c{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-12-05T15:49:25,284 INFO [RS:0;5a29627a94df:34815 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T15:49:25,285 INFO [RS:0;5a29627a94df:34815 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5af9e42e{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-12-05T15:49:25,285 INFO [RS:0;5a29627a94df:34815 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@6e82f00d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,STOPPED} 2024-12-05T15:49:25,286 INFO [regionserver/5a29627a94df:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T15:49:25,286 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T15:49:25,286 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T15:49:25,286 INFO [RS:0;5a29627a94df:34815 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T15:49:25,286 INFO [RS:0;5a29627a94df:34815 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2024-12-05T15:49:25,287 INFO [RS:0;5a29627a94df:34815 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T15:49:25,287 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(3091): Received CLOSE for eb162556a9ba10393a1061e0fd710298 2024-12-05T15:49:25,287 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(3091): Received CLOSE for 94f5b136d6cdc9f73ab4b3495ebd0d3a 2024-12-05T15:49:25,287 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(3091): Received CLOSE for 750a447b5bb19a7405a6a02a816d1329 2024-12-05T15:49:25,287 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(3091): Received CLOSE for d2c0907dfdcc66e63e68a2757d1d51ef 2024-12-05T15:49:25,287 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(3091): Received CLOSE for b7877185d3b50523892995990a3cd728 2024-12-05T15:49:25,287 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(3091): Received CLOSE for 003f376bc19761c728c4bc8dd20d416d 2024-12-05T15:49:25,287 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(3091): Received CLOSE for 24a988b9e3812c25fcebd4f4072c48f0 2024-12-05T15:49:25,287 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(3091): Received CLOSE for 694967a9c2dfe786b3822f9f145f9365 2024-12-05T15:49:25,287 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(3091): Received CLOSE for f422dcfec85c6b145613ea93a68881af 2024-12-05T15:49:25,287 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(959): stopping server 5a29627a94df,34815,1733413637643 2024-12-05T15:49:25,287 INFO [RS:0;5a29627a94df:34815 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T15:49:25,287 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eb162556a9ba10393a1061e0fd710298, disabling compactions & flushes 2024-12-05T15:49:25,287 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. 2024-12-05T15:49:25,287 INFO [RS:0;5a29627a94df:34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;5a29627a94df:34815. 2024-12-05T15:49:25,287 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. 2024-12-05T15:49:25,287 DEBUG [RS:0;5a29627a94df:34815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:25,287 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. after waiting 0 ms 2024-12-05T15:49:25,287 DEBUG [RS:0;5a29627a94df:34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:25,287 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. 2024-12-05T15:49:25,288 INFO [RS:0;5a29627a94df:34815 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T15:49:25,288 INFO [RS:0;5a29627a94df:34815 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T15:49:25,288 INFO [RS:0;5a29627a94df:34815 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T15:49:25,288 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T15:49:25,288 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-12-05T15:49:25,288 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T15:49:25,288 INFO [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T15:49:25,288 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T15:49:25,288 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T15:49:25,288 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T15:49:25,288 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1351): Waiting on 003f376bc19761c728c4bc8dd20d416d, 1588230740, 24a988b9e3812c25fcebd4f4072c48f0, 694967a9c2dfe786b3822f9f145f9365, 750a447b5bb19a7405a6a02a816d1329, 94f5b136d6cdc9f73ab4b3495ebd0d3a, b7877185d3b50523892995990a3cd728, d2c0907dfdcc66e63e68a2757d1d51ef, eb162556a9ba10393a1061e0fd710298, f422dcfec85c6b145613ea93a68881af 2024-12-05T15:49:25,288 INFO [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=23.97 KB heapSize=42.40 KB 2024-12-05T15:49:25,291 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system_bulk/eb162556a9ba10393a1061e0fd710298/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-05T15:49:25,292 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:49:25,292 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. 2024-12-05T15:49:25,292 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eb162556a9ba10393a1061e0fd710298: Waiting for close lock at 1733413765287Running coprocessor pre-close hooks at 1733413765287Disabling compacts and flushes for region at 1733413765287Disabling writes for close at 1733413765287Writing region close event to WAL at 1733413765288 (+1 ms)Running coprocessor post-close hooks at 1733413765292 (+4 ms)Closed at 1733413765292 2024-12-05T15:49:25,292 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1733413659570.eb162556a9ba10393a1061e0fd710298. 2024-12-05T15:49:25,292 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 94f5b136d6cdc9f73ab4b3495ebd0d3a, disabling compactions & flushes 2024-12-05T15:49:25,292 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a. 2024-12-05T15:49:25,292 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a. 2024-12-05T15:49:25,292 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a. after waiting 0 ms 2024-12-05T15:49:25,292 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a. 2024-12-05T15:49:25,296 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/table2/94f5b136d6cdc9f73ab4b3495ebd0d3a/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-12-05T15:49:25,297 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:49:25,297 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a. 2024-12-05T15:49:25,297 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 94f5b136d6cdc9f73ab4b3495ebd0d3a: Waiting for close lock at 1733413765292Running coprocessor pre-close hooks at 1733413765292Disabling compacts and flushes for region at 1733413765292Disabling writes for close at 1733413765292Writing region close event to WAL at 1733413765293 (+1 ms)Running coprocessor post-close hooks at 1733413765297 (+4 ms)Closed at 1733413765297 2024-12-05T15:49:25,297 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:table2,,1733413760246.94f5b136d6cdc9f73ab4b3495ebd0d3a. 2024-12-05T15:49:25,297 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 750a447b5bb19a7405a6a02a816d1329, disabling compactions & flushes 2024-12-05T15:49:25,297 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. 2024-12-05T15:49:25,297 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. 2024-12-05T15:49:25,297 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. after waiting 0 ms 2024-12-05T15:49:25,297 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. 2024-12-05T15:49:25,301 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns3/test-17334136463822/750a447b5bb19a7405a6a02a816d1329/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-05T15:49:25,301 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:49:25,301 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. 2024-12-05T15:49:25,301 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 750a447b5bb19a7405a6a02a816d1329: Waiting for close lock at 1733413765297Running coprocessor pre-close hooks at 1733413765297Disabling compacts and flushes for region at 1733413765297Disabling writes for close at 1733413765297Writing region close event to WAL at 1733413765297Running coprocessor post-close hooks at 1733413765301 (+4 ms)Closed at 1733413765301 2024-12-05T15:49:25,301 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17334136463822,,1733413651593.750a447b5bb19a7405a6a02a816d1329. 2024-12-05T15:49:25,301 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d2c0907dfdcc66e63e68a2757d1d51ef, disabling compactions & flushes 2024-12-05T15:49:25,302 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. 2024-12-05T15:49:25,302 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. 2024-12-05T15:49:25,302 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. after waiting 0 ms 2024-12-05T15:49:25,302 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. 2024-12-05T15:49:25,305 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns4/test-17334136463823/d2c0907dfdcc66e63e68a2757d1d51ef/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-05T15:49:25,306 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:49:25,306 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. 2024-12-05T15:49:25,306 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d2c0907dfdcc66e63e68a2757d1d51ef: Waiting for close lock at 1733413765301Running coprocessor pre-close hooks at 1733413765301Disabling compacts and flushes for region at 1733413765301Disabling writes for close at 1733413765302 (+1 ms)Writing region close event to WAL at 1733413765302Running coprocessor post-close hooks at 1733413765306 (+4 ms)Closed at 1733413765306 2024-12-05T15:49:25,306 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17334136463823,,1733413653759.d2c0907dfdcc66e63e68a2757d1d51ef. 2024-12-05T15:49:25,306 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b7877185d3b50523892995990a3cd728, disabling compactions & flushes 2024-12-05T15:49:25,306 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:49:25,306 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:49:25,306 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. after waiting 0 ms 2024-12-05T15:49:25,306 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:49:25,310 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/ns2/test-17334136463821/b7877185d3b50523892995990a3cd728/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2024-12-05T15:49:25,310 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:49:25,310 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:49:25,310 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b7877185d3b50523892995990a3cd728: Waiting for close lock at 1733413765306Running coprocessor pre-close hooks at 1733413765306Disabling compacts and flushes for region at 1733413765306Disabling writes for close at 1733413765306Writing region close event to WAL at 1733413765307 (+1 ms)Running coprocessor post-close hooks at 1733413765310 (+3 ms)Closed at 1733413765310 2024-12-05T15:49:25,311 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728. 2024-12-05T15:49:25,311 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 003f376bc19761c728c4bc8dd20d416d, disabling compactions & flushes 2024-12-05T15:49:25,311 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:25,311 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:25,311 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. after waiting 0 ms 2024-12-05T15:49:25,311 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:25,312 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/.tmp/info/6e9dc31365c34a0a82d8171119055530 is 159, key is ns2:test-17334136463821,,1733413649275.b7877185d3b50523892995990a3cd728./info:regioninfo/1733413650450/Put/seqid=0 2024-12-05T15:49:25,316 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/003f376bc19761c728c4bc8dd20d416d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=6 2024-12-05T15:49:25,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741996_1172 (size=19093) 2024-12-05T15:49:25,316 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:49:25,317 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:25,317 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 003f376bc19761c728c4bc8dd20d416d: Waiting for close lock at 1733413765311Running coprocessor pre-close hooks at 1733413765311Disabling compacts and flushes for region at 1733413765311Disabling writes for close at 1733413765311Writing region close event to WAL at 1733413765312 (+1 ms)Running coprocessor post-close hooks at 1733413765316 (+4 ms)Closed at 1733413765316 2024-12-05T15:49:25,317 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1733413750760.003f376bc19761c728c4bc8dd20d416d. 2024-12-05T15:49:25,317 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 24a988b9e3812c25fcebd4f4072c48f0, disabling compactions & flushes 2024-12-05T15:49:25,317 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:49:25,317 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:49:25,317 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. after waiting 0 ms 2024-12-05T15:49:25,317 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:49:25,317 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 24a988b9e3812c25fcebd4f4072c48f0 3/3 column families, dataSize=378 B heapSize=1.63 KB 2024-12-05T15:49:25,338 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/f2/c8ad80f143e04d359b107751321534bf is 58, key is row-test-1733413646382-2-0/f2:q1/1733413722337/Put/seqid=0 2024-12-05T15:49:25,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741997_1173 (size=5464) 2024-12-05T15:49:25,489 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 24a988b9e3812c25fcebd4f4072c48f0, 694967a9c2dfe786b3822f9f145f9365, f422dcfec85c6b145613ea93a68881af 2024-12-05T15:49:25,689 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 24a988b9e3812c25fcebd4f4072c48f0, 694967a9c2dfe786b3822f9f145f9365, f422dcfec85c6b145613ea93a68881af 2024-12-05T15:49:25,718 INFO [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.86 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/.tmp/info/6e9dc31365c34a0a82d8171119055530 2024-12-05T15:49:25,740 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/.tmp/ns/84c54568838c472fb2c0c9991d6abd45 is 85, key is ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830./ns:/1733413760248/DeleteFamily/seqid=0 2024-12-05T15:49:25,742 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=378 B at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/f2/c8ad80f143e04d359b107751321534bf 2024-12-05T15:49:25,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741998_1174 (size=5760) 2024-12-05T15:49:25,749 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/.tmp/f2/c8ad80f143e04d359b107751321534bf as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f2/c8ad80f143e04d359b107751321534bf 2024-12-05T15:49:25,753 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/f2/c8ad80f143e04d359b107751321534bf, entries=7, sequenceid=348, filesize=5.3 K 2024-12-05T15:49:25,754 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 24a988b9e3812c25fcebd4f4072c48f0 in 437ms, sequenceid=348, compaction requested=false 2024-12-05T15:49:25,759 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/test-1733413646382/24a988b9e3812c25fcebd4f4072c48f0/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=337 2024-12-05T15:49:25,760 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:49:25,760 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:49:25,760 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 24a988b9e3812c25fcebd4f4072c48f0: Waiting for close lock at 1733413765317Running coprocessor pre-close hooks at 1733413765317Disabling compacts and flushes for region at 1733413765317Disabling writes for close at 1733413765317Obtaining lock to block concurrent updates at 1733413765317Preparing flush snapshotting stores in 24a988b9e3812c25fcebd4f4072c48f0 at 1733413765317Finished memstore snapshotting test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0., syncing WAL and waiting on mvcc, flushsize=dataSize=378, getHeapSize=1616, getOffHeapSize=0, getCellsCount=7 at 1733413765318 (+1 ms)Flushing stores of test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. at 1733413765318Flushing 24a988b9e3812c25fcebd4f4072c48f0/f2: creating writer at 1733413765318Flushing 24a988b9e3812c25fcebd4f4072c48f0/f2: appending metadata at 1733413765337 (+19 ms)Flushing 24a988b9e3812c25fcebd4f4072c48f0/f2: closing flushed file at 1733413765337Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4eb23ae4: reopening flushed file at 1733413765748 (+411 ms)Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 24a988b9e3812c25fcebd4f4072c48f0 in 437ms, sequenceid=348, compaction requested=false at 1733413765754 (+6 ms)Writing region close event to WAL at 1733413765756 (+2 ms)Running coprocessor post-close hooks at 1733413765759 (+3 ms)Closed at 1733413765760 (+1 ms) 2024-12-05T15:49:25,760 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1733413646382,,1733413646872.24a988b9e3812c25fcebd4f4072c48f0. 2024-12-05T15:49:25,760 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 694967a9c2dfe786b3822f9f145f9365, disabling compactions & flushes 2024-12-05T15:49:25,760 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:25,760 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:25,760 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. after waiting 0 ms 2024-12-05T15:49:25,760 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:25,764 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/default/table1/694967a9c2dfe786b3822f9f145f9365/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=6 2024-12-05T15:49:25,764 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:49:25,764 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:25,764 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 694967a9c2dfe786b3822f9f145f9365: Waiting for close lock at 1733413765760Running coprocessor pre-close hooks at 1733413765760Disabling compacts and flushes for region at 1733413765760Disabling writes for close at 1733413765760Writing region close event to WAL at 1733413765761 (+1 ms)Running coprocessor post-close hooks at 1733413765764 (+3 ms)Closed at 1733413765764 2024-12-05T15:49:25,764 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,row0,1733413750760.694967a9c2dfe786b3822f9f145f9365. 2024-12-05T15:49:25,765 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f422dcfec85c6b145613ea93a68881af, disabling compactions & flushes 2024-12-05T15:49:25,765 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:49:25,765 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:49:25,765 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. after waiting 0 ms 2024-12-05T15:49:25,765 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:49:25,765 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f422dcfec85c6b145613ea93a68881af 2/2 column families, dataSize=2.12 KB heapSize=3.43 KB 2024-12-05T15:49:25,768 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/meta/fdd94a8382414ddabebe699e53b80383 is 158, key is trslm:hdfs://localhost:37233/backupUT\x00ns2:test-17334136463821/meta:log-roll-map/1733413739566/Put/seqid=0 2024-12-05T15:49:25,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741999_1175 (size=6208) 2024-12-05T15:49:25,889 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f422dcfec85c6b145613ea93a68881af 2024-12-05T15:49:25,949 INFO [regionserver/5a29627a94df:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-05T15:49:25,949 INFO [regionserver/5a29627a94df:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-05T15:49:26,089 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f422dcfec85c6b145613ea93a68881af 2024-12-05T15:49:26,146 INFO [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=474 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/.tmp/ns/84c54568838c472fb2c0c9991d6abd45 2024-12-05T15:49:26,170 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/.tmp/rep_barrier/0680197a94d84f8f8501c296fbc36d97 is 94, key is ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830./rep_barrier:/1733413760248/DeleteFamily/seqid=0 2024-12-05T15:49:26,172 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=944 B at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/meta/fdd94a8382414ddabebe699e53b80383 2024-12-05T15:49:26,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073742000_1176 (size=5640) 2024-12-05T15:49:26,180 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/session/5f321a8a0f6f4841b53eae517d17c6f4 is 588, key is session:backup_1733413727346/session:context/1733413740004/Put/seqid=0 2024-12-05T15:49:26,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073742001_1177 (size=5689) 2024-12-05T15:49:26,290 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-05T15:49:26,290 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, f422dcfec85c6b145613ea93a68881af=backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af.} 2024-12-05T15:49:26,290 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f422dcfec85c6b145613ea93a68881af 2024-12-05T15:49:26,490 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f422dcfec85c6b145613ea93a68881af 2024-12-05T15:49:26,572 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2024-12-05T15:49:26,573 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-12-05T15:49:26,573 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1733413646382 2024-12-05T15:49:26,573 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17334136463823 2024-12-05T15:49:26,573 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17334136463822 2024-12-05T15:49:26,573 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17334136463821 2024-12-05T15:49:26,573 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2024-12-05T15:49:26,577 INFO [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=266 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/.tmp/rep_barrier/0680197a94d84f8f8501c296fbc36d97 2024-12-05T15:49:26,586 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.20 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/session/5f321a8a0f6f4841b53eae517d17c6f4 2024-12-05T15:49:26,593 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/meta/fdd94a8382414ddabebe699e53b80383 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/fdd94a8382414ddabebe699e53b80383 2024-12-05T15:49:26,599 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/meta/fdd94a8382414ddabebe699e53b80383, entries=7, sequenceid=52, filesize=6.1 K 2024-12-05T15:49:26,600 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/.tmp/session/5f321a8a0f6f4841b53eae517d17c6f4 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/5f321a8a0f6f4841b53eae517d17c6f4 2024-12-05T15:49:26,605 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/session/5f321a8a0f6f4841b53eae517d17c6f4, entries=2, sequenceid=52, filesize=5.6 K 2024-12-05T15:49:26,606 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~2.12 KB/2170, heapSize ~3.40 KB/3480, currentSize=0 B/0 for f422dcfec85c6b145613ea93a68881af in 841ms, sequenceid=52, compaction requested=true 2024-12-05T15:49:26,607 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/.tmp/table/d7ce9049591e4cf0a48a5cd3a0682ae3 is 88, key is ns2:table2,,1733413745772.4c2f6298605fad0624d0dec8fedda830./table:/1733413760248/DeleteFamily/seqid=0 2024-12-05T15:49:26,611 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/backup/system/f422dcfec85c6b145613ea93a68881af/recovered.edits/55.seqid, newMaxSeqId=55, maxSeqId=1 2024-12-05T15:49:26,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073742002_1178 (size=6364) 2024-12-05T15:49:26,611 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:49:26,611 INFO [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:49:26,611 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f422dcfec85c6b145613ea93a68881af: Waiting for close lock at 1733413765765Running coprocessor pre-close hooks at 1733413765765Disabling compacts and flushes for region at 1733413765765Disabling writes for close at 1733413765765Obtaining lock to block concurrent updates at 1733413765765Preparing flush snapshotting stores in f422dcfec85c6b145613ea93a68881af at 1733413765765Finished memstore snapshotting backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af., syncing WAL and waiting on mvcc, flushsize=dataSize=2170, getHeapSize=3480, getOffHeapSize=0, getCellsCount=11 at 1733413765765Flushing stores of backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. at 1733413765766 (+1 ms)Flushing f422dcfec85c6b145613ea93a68881af/meta: creating writer at 1733413765766Flushing f422dcfec85c6b145613ea93a68881af/meta: appending metadata at 1733413765768 (+2 ms)Flushing f422dcfec85c6b145613ea93a68881af/meta: closing flushed file at 1733413765768Flushing f422dcfec85c6b145613ea93a68881af/session: creating writer at 1733413766177 (+409 ms)Flushing f422dcfec85c6b145613ea93a68881af/session: appending metadata at 1733413766180 (+3 ms)Flushing f422dcfec85c6b145613ea93a68881af/session: closing flushed file at 1733413766180Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6503b8d5: reopening flushed file at 1733413766592 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a85b30f: reopening flushed file at 1733413766599 (+7 ms)Finished flush of dataSize ~2.12 KB/2170, heapSize ~3.40 KB/3480, currentSize=0 B/0 for f422dcfec85c6b145613ea93a68881af in 841ms, sequenceid=52, compaction requested=true at 1733413766606 (+7 ms)Writing region close event to WAL at 1733413766607 (+1 ms)Running coprocessor post-close hooks at 1733413766611 (+4 ms)Closed at 1733413766611 2024-12-05T15:49:26,612 DEBUG [RS_CLOSE_REGION-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1733413657398.f422dcfec85c6b145613ea93a68881af. 2024-12-05T15:49:26,690 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T15:49:26,891 DEBUG [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T15:49:27,012 INFO [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.39 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/.tmp/table/d7ce9049591e4cf0a48a5cd3a0682ae3 2024-12-05T15:49:27,024 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/.tmp/info/6e9dc31365c34a0a82d8171119055530 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/info/6e9dc31365c34a0a82d8171119055530 2024-12-05T15:49:27,030 INFO [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/info/6e9dc31365c34a0a82d8171119055530, entries=114, sequenceid=97, filesize=18.6 K 2024-12-05T15:49:27,031 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/.tmp/ns/84c54568838c472fb2c0c9991d6abd45 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/ns/84c54568838c472fb2c0c9991d6abd45 2024-12-05T15:49:27,035 INFO [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/ns/84c54568838c472fb2c0c9991d6abd45, entries=10, sequenceid=97, filesize=5.6 K 2024-12-05T15:49:27,036 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/.tmp/rep_barrier/0680197a94d84f8f8501c296fbc36d97 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/rep_barrier/0680197a94d84f8f8501c296fbc36d97 2024-12-05T15:49:27,041 INFO [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/rep_barrier/0680197a94d84f8f8501c296fbc36d97, entries=3, sequenceid=97, filesize=5.5 K 2024-12-05T15:49:27,042 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/.tmp/table/d7ce9049591e4cf0a48a5cd3a0682ae3 as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/table/d7ce9049591e4cf0a48a5cd3a0682ae3 2024-12-05T15:49:27,046 INFO [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/table/d7ce9049591e4cf0a48a5cd3a0682ae3, entries=21, sequenceid=97, filesize=6.2 K 2024-12-05T15:49:27,047 INFO [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~23.97 KB/24544, heapSize ~42.34 KB/43352, currentSize=0 B/0 for 1588230740 in 1759ms, sequenceid=97, compaction requested=false 2024-12-05T15:49:27,051 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/data/hbase/meta/1588230740/recovered.edits/100.seqid, newMaxSeqId=100, maxSeqId=1 2024-12-05T15:49:27,052 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-05T15:49:27,052 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T15:49:27,052 INFO [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T15:49:27,052 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733413765288Running coprocessor pre-close hooks at 1733413765288Disabling compacts and flushes for region at 1733413765288Disabling writes for close at 1733413765288Obtaining lock to block concurrent updates at 1733413765288Preparing flush snapshotting stores in 1588230740 at 1733413765288Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=24544, getHeapSize=43352, getOffHeapSize=0, getCellsCount=235 at 1733413765289 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733413765289Flushing 1588230740/info: creating writer at 1733413765289Flushing 1588230740/info: appending metadata at 1733413765312 (+23 ms)Flushing 1588230740/info: closing flushed file at 1733413765312Flushing 1588230740/ns: creating writer at 1733413765727 (+415 ms)Flushing 1588230740/ns: appending metadata at 1733413765740 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1733413765740Flushing 1588230740/rep_barrier: creating writer at 1733413766157 (+417 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733413766170 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733413766170Flushing 1588230740/table: creating writer at 1733413766589 (+419 ms)Flushing 1588230740/table: appending metadata at 1733413766606 (+17 ms)Flushing 1588230740/table: closing flushed file at 1733413766607 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ad295f5: reopening flushed file at 1733413767022 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11538d42: reopening flushed file at 1733413767030 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f64479d: reopening flushed file at 1733413767036 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@601baeb: reopening flushed file at 1733413767041 (+5 ms)Finished flush of dataSize ~23.97 KB/24544, heapSize ~42.34 KB/43352, currentSize=0 B/0 for 1588230740 in 1759ms, sequenceid=97, compaction requested=false at 1733413767047 (+6 ms)Writing region close event to WAL at 1733413767048 (+1 ms)Running coprocessor post-close hooks at 1733413767052 (+4 ms)Closed at 1733413767052 2024-12-05T15:49:27,052 DEBUG [RS_CLOSE_META-regionserver/5a29627a94df:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T15:49:27,091 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(976): stopping server 5a29627a94df,34815,1733413637643; all regions closed. 2024-12-05T15:49:27,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741956_1132 (size=16160) 2024-12-05T15:49:27,095 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.meta.1733413731917.meta not finished, retry = 0 2024-12-05T15:49:27,206 DEBUG [RS:0;5a29627a94df:34815 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs 2024-12-05T15:49:27,206 INFO [RS:0;5a29627a94df:34815 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5a29627a94df%2C34815%2C1733413637643.meta:.meta(num 1733413731917) 2024-12-05T15:49:27,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741955_1131 (size=4165) 2024-12-05T15:49:27,210 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/WALs/5a29627a94df,34815,1733413637643/5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0.1733413731902 not finished, retry = 0 2024-12-05T15:49:27,314 DEBUG [RS:0;5a29627a94df:34815 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs 2024-12-05T15:49:27,314 INFO [RS:0;5a29627a94df:34815 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-0:(num 1733413731902) 2024-12-05T15:49:27,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741954_1130 (size=7883) 2024-12-05T15:49:27,321 DEBUG [RS:0;5a29627a94df:34815 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/oldWALs 2024-12-05T15:49:27,321 INFO [RS:0;5a29627a94df:34815 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5a29627a94df%2C34815%2C1733413637643.5a29627a94df%2C34815%2C1733413637643.regiongroup-1:(num 1733413731883) 2024-12-05T15:49:27,321 DEBUG [RS:0;5a29627a94df:34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:27,321 INFO [RS:0;5a29627a94df:34815 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T15:49:27,322 INFO [RS:0;5a29627a94df:34815 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T15:49:27,322 INFO [RS:0;5a29627a94df:34815 {}] hbase.ChoreService(370): Chore service for: regionserver/5a29627a94df:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T15:49:27,322 INFO [RS:0;5a29627a94df:34815 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T15:49:27,322 INFO [regionserver/5a29627a94df:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T15:49:27,323 INFO [RS:0;5a29627a94df:34815 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34815 2024-12-05T15:49:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-12-05T15:49:27,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/5a29627a94df,34815,1733413637643 2024-12-05T15:49:27,364 INFO [RS:0;5a29627a94df:34815 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T15:49:27,372 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5a29627a94df,34815,1733413637643] 2024-12-05T15:49:27,380 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /1/draining/5a29627a94df,34815,1733413637643 already deleted, retry=false 2024-12-05T15:49:27,380 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5a29627a94df,34815,1733413637643 expired; onlineServers=0 2024-12-05T15:49:27,380 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '5a29627a94df,33117,1733413636177' ***** 2024-12-05T15:49:27,381 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T15:49:27,381 INFO [M:0;5a29627a94df:33117 {}] hbase.HBaseServerBase(468): Stop info server 2024-12-05T15:49:27,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2024-12-05T15:49:27,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-05T15:49:27,391 INFO [M:0;5a29627a94df:33117 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@77e4d37e{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-12-05T15:49:27,392 INFO [M:0;5a29627a94df:33117 {}] server.AbstractConnector(383): Stopped ServerConnector@1ab434da{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-12-05T15:49:27,392 INFO [M:0;5a29627a94df:33117 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T15:49:27,392 INFO [M:0;5a29627a94df:33117 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@ebfbfd2{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-12-05T15:49:27,392 INFO [M:0;5a29627a94df:33117 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@4acbd630{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,STOPPED} 2024-12-05T15:49:27,392 INFO [M:0;5a29627a94df:33117 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T15:49:27,393 INFO [M:0;5a29627a94df:33117 {}] master.HMaster(1770): Stopping master jetty server 2024-12-05T15:49:27,393 INFO [M:0;5a29627a94df:33117 {}] server.AbstractConnector(383): Stopped ServerConnector@5207b9f1{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-12-05T15:49:27,393 INFO [M:0;5a29627a94df:33117 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T15:49:27,394 DEBUG [M:0;5a29627a94df:33117 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T15:49:27,394 INFO [M:0;5a29627a94df:33117 {}] master.BackupLogCleaner(194): Stopping BackupLogCleaner 2024-12-05T15:49:27,394 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T15:49:27,394 INFO [M:0;5a29627a94df:33117 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;5a29627a94df:33117. 2024-12-05T15:49:27,394 DEBUG [master/5a29627a94df:0:becomeActiveMaster-HFileCleaner.small.0-1733413640736 {}] cleaner.HFileCleaner(306): Exit Thread[master/5a29627a94df:0:becomeActiveMaster-HFileCleaner.small.0-1733413640736,5,FailOnTimeoutGroup] 2024-12-05T15:49:27,394 DEBUG [master/5a29627a94df:0:becomeActiveMaster-HFileCleaner.large.0-1733413640735 {}] cleaner.HFileCleaner(306): Exit Thread[master/5a29627a94df:0:becomeActiveMaster-HFileCleaner.large.0-1733413640735,5,FailOnTimeoutGroup] 2024-12-05T15:49:27,394 DEBUG [M:0;5a29627a94df:33117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupHFileCleaner.stop(BackupHFileCleaner.java:156) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.cleanup(CleanerChore.java:444) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.cleanup(HFileCleaner.java:239) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:333) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:322) at org.apache.hadoop.hbase.master.HMaster.stopChores(HMaster.java:1923) at org.apache.hadoop.hbase.HBaseServerBase.stopChoreService(HBaseServerBase.java:439) at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1777) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T15:49:27,394 DEBUG [M:0;5a29627a94df:33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:27,394 DEBUG [M:0;5a29627a94df:33117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T15:49:27,394 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T15:49:27,394 DEBUG [M:0;5a29627a94df:33117 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T15:49:27,394 INFO [M:0;5a29627a94df:33117 {}] hbase.ChoreService(370): Chore service for: master/5a29627a94df:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T15:49:27,394 INFO [M:0;5a29627a94df:33117 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T15:49:27,394 DEBUG [M:0;5a29627a94df:33117 {}] master.HMaster(1795): Stopping service threads 2024-12-05T15:49:27,394 INFO [M:0;5a29627a94df:33117 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T15:49:27,395 INFO [M:0;5a29627a94df:33117 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T15:49:27,395 ERROR [M:0;5a29627a94df:33117 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-1,5,PEWorkerGroup] Thread[HFileArchiver-2,5,PEWorkerGroup] Thread[HFileArchiver-3,5,PEWorkerGroup] 2024-12-05T15:49:27,395 INFO [M:0;5a29627a94df:33117 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T15:49:27,395 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T15:49:27,396 DEBUG [M:0;5a29627a94df:33117 {}] zookeeper.ZKUtil(347): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2024-12-05T15:49:27,396 WARN [M:0;5a29627a94df:33117 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T15:49:27,396 INFO [M:0;5a29627a94df:33117 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/.lastflushedseqids 2024-12-05T15:49:27,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073742003_1179 (size=761) 2024-12-05T15:49:27,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T15:49:27,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x101a75acab50001, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T15:49:27,473 INFO [RS:0;5a29627a94df:34815 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T15:49:27,473 INFO [RS:0;5a29627a94df:34815 {}] regionserver.HRegionServer(1031): Exiting; stopping=5a29627a94df,34815,1733413637643; zookeeper connection closed. 2024-12-05T15:49:27,474 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@27d601d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@27d601d 2024-12-05T15:49:27,474 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T15:49:27,807 INFO [M:0;5a29627a94df:33117 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T15:49:27,808 INFO [M:0;5a29627a94df:33117 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T15:49:27,808 DEBUG [M:0;5a29627a94df:33117 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T15:49:27,826 INFO [M:0;5a29627a94df:33117 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T15:49:27,826 DEBUG [M:0;5a29627a94df:33117 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T15:49:27,826 DEBUG [M:0;5a29627a94df:33117 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T15:49:27,826 DEBUG [M:0;5a29627a94df:33117 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T15:49:27,826 INFO [M:0;5a29627a94df:33117 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=441.44 KB heapSize=518.47 KB 2024-12-05T15:49:27,839 DEBUG [M:0;5a29627a94df:33117 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/56ed5a84e9424983990224aaa6ddfc8b is 82, key is hbase:meta,,1/info:regioninfo/1733413642064/Put/seqid=0 2024-12-05T15:49:27,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073742004_1180 (size=5672) 2024-12-05T15:49:28,245 INFO [M:0;5a29627a94df:33117 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=907 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/56ed5a84e9424983990224aaa6ddfc8b 2024-12-05T15:49:28,272 DEBUG [M:0;5a29627a94df:33117 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4e7f2041806541d5ab60e5eca5b901fb is 2201, key is \x00\x00\x00\x00\x00\x00\x00Q/proc:d/1733413757478/Put/seqid=0 2024-12-05T15:49:28,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073742005_1181 (size=30172) 2024-12-05T15:49:28,678 INFO [M:0;5a29627a94df:33117 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=440.88 KB at sequenceid=907 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4e7f2041806541d5ab60e5eca5b901fb 2024-12-05T15:49:28,689 INFO [M:0;5a29627a94df:33117 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4e7f2041806541d5ab60e5eca5b901fb 2024-12-05T15:49:28,704 DEBUG [M:0;5a29627a94df:33117 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6e201b53a1944e81a525615c87c622ba is 69, key is 5a29627a94df,34815,1733413637643/rs:state/1733413640850/Put/seqid=0 2024-12-05T15:49:28,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073742006_1182 (size=5156) 2024-12-05T15:49:29,110 INFO [M:0;5a29627a94df:33117 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=907 (bloomFilter=true), to=hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6e201b53a1944e81a525615c87c622ba 2024-12-05T15:49:29,123 DEBUG [M:0;5a29627a94df:33117 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/56ed5a84e9424983990224aaa6ddfc8b as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/56ed5a84e9424983990224aaa6ddfc8b 2024-12-05T15:49:29,131 INFO [M:0;5a29627a94df:33117 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/56ed5a84e9424983990224aaa6ddfc8b, entries=8, sequenceid=907, filesize=5.5 K 2024-12-05T15:49:29,132 DEBUG [M:0;5a29627a94df:33117 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4e7f2041806541d5ab60e5eca5b901fb as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4e7f2041806541d5ab60e5eca5b901fb 2024-12-05T15:49:29,136 INFO [M:0;5a29627a94df:33117 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4e7f2041806541d5ab60e5eca5b901fb 2024-12-05T15:49:29,136 INFO [M:0;5a29627a94df:33117 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4e7f2041806541d5ab60e5eca5b901fb, entries=94, sequenceid=907, filesize=29.5 K 2024-12-05T15:49:29,137 DEBUG [M:0;5a29627a94df:33117 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6e201b53a1944e81a525615c87c622ba as hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6e201b53a1944e81a525615c87c622ba 2024-12-05T15:49:29,142 INFO [M:0;5a29627a94df:33117 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37233/user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6e201b53a1944e81a525615c87c622ba, entries=1, sequenceid=907, filesize=5.0 K 2024-12-05T15:49:29,143 INFO [M:0;5a29627a94df:33117 {}] regionserver.HRegion(3140): Finished flush of dataSize ~441.44 KB/452034, heapSize ~518.17 KB/530608, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1317ms, sequenceid=907, compaction requested=false 2024-12-05T15:49:29,144 INFO [M:0;5a29627a94df:33117 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T15:49:29,144 DEBUG [M:0;5a29627a94df:33117 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733413767808Disabling compacts and flushes for region at 1733413767808Disabling writes for close at 1733413767826 (+18 ms)Obtaining lock to block concurrent updates at 1733413767826Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733413767826Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=452034, getHeapSize=530848, getOffHeapSize=0, getCellsCount=1030 at 1733413767826Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733413767827 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733413767827Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733413767839 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733413767839Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733413768256 (+417 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733413768272 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733413768272Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733413768690 (+418 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733413768704 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733413768704Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a02abb3: reopening flushed file at 1733413769121 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16c9cfbd: reopening flushed file at 1733413769131 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f9aa215: reopening flushed file at 1733413769137 (+6 ms)Finished flush of dataSize ~441.44 KB/452034, heapSize ~518.17 KB/530608, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1317ms, sequenceid=907, compaction requested=false at 1733413769143 (+6 ms)Writing region close event to WAL at 1733413769144 (+1 ms)Closed at 1733413769144 2024-12-05T15:49:29,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43303 is added to blk_1073741830_1006 (size=511900) 2024-12-05T15:49:29,147 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/38bd491d-4a4b-9b84-1a2d-c11ddf3a3089/MasterData/WALs/5a29627a94df,33117,1733413636177/5a29627a94df%2C33117%2C1733413636177.5a29627a94df%2C33117%2C1733413636177.regiongroup-0.1733413639926 not finished, retry = 0 2024-12-05T15:49:29,249 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T15:49:29,249 INFO [M:0;5a29627a94df:33117 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T15:49:29,249 INFO [M:0;5a29627a94df:33117 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2024-12-05T15:49:29,249 INFO [M:0;5a29627a94df:33117 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33117 2024-12-05T15:49:29,250 INFO [M:0;5a29627a94df:33117 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T15:49:29,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T15:49:29,397 INFO [M:0;5a29627a94df:33117 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T15:49:29,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33117-0x101a75acab50000, quorum=127.0.0.1:50094, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T15:49:29,415 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d3c17d4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T15:49:29,418 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@575d15e0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T15:49:29,418 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T15:49:29,418 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@308dcebb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T15:49:29,418 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@347c3eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,STOPPED} 2024-12-05T15:49:29,421 WARN [BP-671471983-172.17.0.3-1733413632680 heartbeating to localhost/127.0.0.1:37233 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T15:49:29,421 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T15:49:29,421 WARN [BP-671471983-172.17.0.3-1733413632680 heartbeating to localhost/127.0.0.1:37233 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-671471983-172.17.0.3-1733413632680 (Datanode Uuid 8025ece3-eea0-4ad5-b9b3-0c5f4a722f8c) service to localhost/127.0.0.1:37233 2024-12-05T15:49:29,421 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T15:49:29,422 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/cluster_783e9d64-ce48-dc27-a242-83d4c51d286c/data/data1/current/BP-671471983-172.17.0.3-1733413632680 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T15:49:29,423 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/cluster_783e9d64-ce48-dc27-a242-83d4c51d286c/data/data2/current/BP-671471983-172.17.0.3-1733413632680 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T15:49:29,423 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T15:49:29,428 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1d57f2ce{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T15:49:29,429 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@382e7786{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T15:49:29,429 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T15:49:29,429 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@325f8c88{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T15:49:29,429 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@205baae5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,STOPPED} 2024-12-05T15:49:29,437 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T15:49:29,496 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-05T15:49:29,496 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2377): Stopping mini mapreduce cluster... 2024-12-05T15:49:29,501 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@333abf77{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-05T15:49:29,501 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3ede3ae6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T15:49:29,502 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T15:49:29,502 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78ebccb1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-05T15:49:29,502 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31dc727{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,STOPPED} 2024-12-05T15:49:29,509 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@30e0ce95{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-05T15:49:29,510 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5668db5e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T15:49:29,510 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T15:49:29,510 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@591f9edc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-05T15:49:29,510 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a9ede50{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,STOPPED} 2024-12-05T15:49:29,513 ERROR [Thread[Thread-270,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-12-05T15:49:29,513 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@67e7621b{cluster,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-12-05T15:49:29,514 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3f7727e1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T15:49:29,514 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T15:49:29,514 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336bed82{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-05T15:49:29,514 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ef40a45{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,STOPPED} 2024-12-05T15:49:29,516 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2024-12-05T15:49:29,519 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(72): Returning, interrupted : java.lang.InterruptedException 2024-12-05T15:49:29,519 ERROR [ResourceManager Event Processor Monitor {}] resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessorMonitor(1193): Returning, interrupted : java.lang.InterruptedException: sleep interrupted 2024-12-05T15:49:29,520 ERROR [Thread[Thread-293,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-12-05T15:49:29,523 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2d6554c9{jobhistory,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-12-05T15:49:29,523 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34d2c2ee{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T15:49:29,523 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T15:49:29,524 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71ff468e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-05T15:49:29,524 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a344172{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/8d2befa9-eac8-47ad-35d5-b67469ae7986/hadoop.log.dir/,STOPPED} 2024-12-05T15:49:29,525 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-12-05T15:49:29,525 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2380): Mini mapreduce cluster stopped