2024-11-24 04:21:27,988 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-24 04:21:28,001 main DEBUG Took 0.010427 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 04:21:28,002 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 04:21:28,002 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 04:21:28,003 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 04:21:28,004 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:21:28,014 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 04:21:28,025 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:21:28,027 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:21:28,027 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:21:28,028 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:21:28,028 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:21:28,028 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:21:28,029 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:21:28,030 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:21:28,030 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:21:28,030 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:21:28,031 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:21:28,031 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:21:28,032 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:21:28,032 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:21:28,033 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:21:28,033 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:21:28,033 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:21:28,034 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:21:28,034 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:21:28,034 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:21:28,035 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:21:28,035 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:21:28,035 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:21:28,036 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:21:28,036 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:21:28,036 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 04:21:28,038 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:21:28,039 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 04:21:28,041 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 04:21:28,041 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 04:21:28,042 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 04:21:28,043 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 04:21:28,051 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 04:21:28,054 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 04:21:28,055 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 04:21:28,056 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 04:21:28,056 main DEBUG createAppenders(={Console}) 2024-11-24 04:21:28,057 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 initialized 2024-11-24 04:21:28,057 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-24 04:21:28,058 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 OK. 2024-11-24 04:21:28,058 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 04:21:28,058 main DEBUG OutputStream closed 2024-11-24 04:21:28,059 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 04:21:28,059 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 04:21:28,059 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3c73951 OK 2024-11-24 04:21:28,128 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 04:21:28,130 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 04:21:28,131 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 04:21:28,132 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 04:21:28,133 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 04:21:28,133 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 04:21:28,133 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 04:21:28,134 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 04:21:28,134 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 04:21:28,134 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 04:21:28,135 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 04:21:28,135 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 04:21:28,135 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 04:21:28,136 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 04:21:28,136 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 04:21:28,136 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 04:21:28,136 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 04:21:28,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 04:21:28,139 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 04:21:28,140 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@61fe30) with optional ClassLoader: null 2024-11-24 04:21:28,140 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 04:21:28,141 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@61fe30] started OK. 2024-11-24T04:21:28,175 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackup timeout: 13 mins 2024-11-24 04:21:28,177 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 04:21:28,178 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T04:21:28,568 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef 2024-11-24T04:21:28,573 DEBUG [Time-limited test {}] impl.BackupManager(124): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-24T04:21:28,576 DEBUG [Time-limited test {}] impl.BackupManager(154): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:21:28,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T04:21:28,596 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/cluster_bf592bf1-aea9-8622-9295-bd02d49cdf8d, deleteOnExit=true 2024-11-24T04:21:28,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T04:21:28,598 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/test.cache.data in system properties and HBase conf 2024-11-24T04:21:28,599 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T04:21:28,600 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir in system properties and HBase conf 2024-11-24T04:21:28,600 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T04:21:28,601 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T04:21:28,601 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T04:21:28,688 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T04:21:28,780 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T04:21:28,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T04:21:28,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T04:21:28,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T04:21:28,787 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T04:21:28,787 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T04:21:28,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T04:21:28,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T04:21:28,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T04:21:28,790 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T04:21:28,790 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/nfs.dump.dir in system properties and HBase conf 2024-11-24T04:21:28,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/java.io.tmpdir in system properties and HBase conf 2024-11-24T04:21:28,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T04:21:28,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T04:21:28,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T04:21:29,880 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T04:21:29,967 INFO [Time-limited test {}] log.Log(170): Logging initialized @2774ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T04:21:30,055 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:21:30,143 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:21:30,174 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:21:30,175 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:21:30,177 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T04:21:30,194 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:21:30,198 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3cb0c733{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:21:30,199 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@282f8860{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T04:21:30,400 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c784fd0{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/java.io.tmpdir/jetty-localhost-45813-hadoop-hdfs-3_4_1-tests_jar-_-any-13244419624805380229/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T04:21:30,408 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2fdb04ed{HTTP/1.1, (http/1.1)}{localhost:45813} 2024-11-24T04:21:30,408 INFO [Time-limited test {}] server.Server(415): Started @3216ms 2024-11-24T04:21:30,971 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:21:30,981 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:21:30,983 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:21:30,983 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:21:30,983 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T04:21:30,984 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@526a6450{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:21:30,985 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8cd91a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T04:21:31,100 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@cb44ad4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/java.io.tmpdir/jetty-localhost-41309-hadoop-hdfs-3_4_1-tests_jar-_-any-8787259348791450233/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:21:31,101 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d3ca681{HTTP/1.1, (http/1.1)}{localhost:41309} 2024-11-24T04:21:31,101 INFO [Time-limited test {}] server.Server(415): Started @3909ms 2024-11-24T04:21:31,161 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T04:21:32,228 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/cluster_bf592bf1-aea9-8622-9295-bd02d49cdf8d/data/data1/current/BP-1911929303-172.17.0.3-1732422089355/current, will proceed with Du for space computation calculation, 2024-11-24T04:21:32,228 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/cluster_bf592bf1-aea9-8622-9295-bd02d49cdf8d/data/data2/current/BP-1911929303-172.17.0.3-1732422089355/current, will proceed with Du for space computation calculation, 2024-11-24T04:21:32,256 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T04:21:32,302 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x101921c952f40e2d with lease ID 0x59b1f5d8d2f8b2de: Processing first storage report for DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c from datanode DatanodeRegistration(127.0.0.1:44025, datanodeUuid=b485fe1d-3b22-4447-85b3-8150d5328dec, infoPort=40087, infoSecurePort=0, ipcPort=35741, storageInfo=lv=-57;cid=testClusterID;nsid=192268152;c=1732422089355) 2024-11-24T04:21:32,303 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x101921c952f40e2d with lease ID 0x59b1f5d8d2f8b2de: from storage DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c node DatanodeRegistration(127.0.0.1:44025, datanodeUuid=b485fe1d-3b22-4447-85b3-8150d5328dec, infoPort=40087, infoSecurePort=0, ipcPort=35741, storageInfo=lv=-57;cid=testClusterID;nsid=192268152;c=1732422089355), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-24T04:21:32,304 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x101921c952f40e2d with lease ID 0x59b1f5d8d2f8b2de: Processing first storage report for DS-23a346d2-0bf1-4896-9549-d52dbf218476 from datanode DatanodeRegistration(127.0.0.1:44025, datanodeUuid=b485fe1d-3b22-4447-85b3-8150d5328dec, infoPort=40087, infoSecurePort=0, ipcPort=35741, storageInfo=lv=-57;cid=testClusterID;nsid=192268152;c=1732422089355) 2024-11-24T04:21:32,304 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x101921c952f40e2d with lease ID 0x59b1f5d8d2f8b2de: from storage DS-23a346d2-0bf1-4896-9549-d52dbf218476 node DatanodeRegistration(127.0.0.1:44025, datanodeUuid=b485fe1d-3b22-4447-85b3-8150d5328dec, infoPort=40087, infoSecurePort=0, ipcPort=35741, storageInfo=lv=-57;cid=testClusterID;nsid=192268152;c=1732422089355), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T04:21:32,384 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef 2024-11-24T04:21:32,534 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/cluster_bf592bf1-aea9-8622-9295-bd02d49cdf8d/zookeeper_0, clientPort=64163, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/cluster_bf592bf1-aea9-8622-9295-bd02d49cdf8d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/cluster_bf592bf1-aea9-8622-9295-bd02d49cdf8d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T04:21:32,547 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64163 2024-11-24T04:21:32,562 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:21:32,566 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:21:32,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741825_1001 (size=7) 2024-11-24T04:21:33,226 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd with version=8 2024-11-24T04:21:33,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging 2024-11-24T04:21:33,237 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(136): Setting Master Port to random. 2024-11-24T04:21:33,237 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(143): Setting RegionServer Port to random. 2024-11-24T04:21:33,237 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(153): Setting RS InfoServer Port to random. 2024-11-24T04:21:33,237 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(161): Setting Master InfoServer Port to random. 2024-11-24T04:21:33,309 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T04:21:33,551 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e57b7116ff5e:0 server-side Connection retries=45 2024-11-24T04:21:33,560 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:21:33,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T04:21:33,565 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T04:21:33,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:21:33,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T04:21:33,723 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T04:21:33,782 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T04:21:33,792 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T04:21:33,795 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T04:21:33,819 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 159794 (auto-detected) 2024-11-24T04:21:33,820 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-24T04:21:33,836 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46719 2024-11-24T04:21:33,857 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46719 connecting to ZooKeeper ensemble=127.0.0.1:64163 2024-11-24T04:21:33,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:467190x0, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T04:21:33,973 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46719-0x1016b1342860000 connected 2024-11-24T04:21:34,055 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:21:34,058 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:21:34,071 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-24T04:21:34,100 INFO [Time-limited test {}] log.Log(170): Logging initialized @6908ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T04:21:34,158 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-24T04:21:34,159 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-24T04:21:34,160 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-24T04:21:34,160 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-24T04:21:34,161 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-24T04:21:34,163 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2024-11-24T04:21:34,164 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-24T04:21:34,164 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-24T04:21:34,166 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-24T04:21:34,238 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 36905 2024-11-24T04:21:34,239 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-24T04:21:34,251 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-24T04:21:34,252 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@7c1597dd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:21:34,253 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-24T04:21:34,253 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@65507143{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-24T04:21:34,293 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-24T04:21:34,300 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:21:34,300 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:21:34,301 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T04:21:34,305 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-24T04:21:34,893 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@5c685ba8{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-11-24T04:21:34,897 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10f9ba48{HTTP/1.1, (http/1.1)}{0.0.0.0:36905} 2024-11-24T04:21:34,897 INFO [Time-limited test {}] server.Server(415): Started @7705ms 2024-11-24T04:21:34,900 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd, hbase.cluster.distributed=false 2024-11-24T04:21:34,922 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-24T04:21:34,928 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46719 2024-11-24T04:21:34,929 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46719 2024-11-24T04:21:34,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46719 2024-11-24T04:21:34,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46719 2024-11-24T04:21:34,932 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46719 2024-11-24T04:21:35,042 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e57b7116ff5e:0 server-side Connection retries=45 2024-11-24T04:21:35,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:21:35,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T04:21:35,044 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T04:21:35,045 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:21:35,045 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T04:21:35,048 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T04:21:35,052 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T04:21:35,053 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36347 2024-11-24T04:21:35,055 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36347 connecting to ZooKeeper ensemble=127.0.0.1:64163 2024-11-24T04:21:35,057 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:21:35,060 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:21:35,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:363470x0, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T04:21:35,125 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36347-0x1016b1342860001 connected 2024-11-24T04:21:35,125 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:363470x0, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-24T04:21:35,128 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-24T04:21:35,128 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-24T04:21:35,128 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-24T04:21:35,129 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-24T04:21:35,129 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-24T04:21:35,130 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2024-11-24T04:21:35,130 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-24T04:21:35,130 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-24T04:21:35,131 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-24T04:21:35,133 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 39651 2024-11-24T04:21:35,133 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-24T04:21:35,135 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-24T04:21:35,136 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3e9615b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:21:35,136 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-24T04:21:35,136 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@2df3a35c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-24T04:21:35,154 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-24T04:21:35,156 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:21:35,156 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:21:35,156 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T04:21:35,158 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-24T04:21:35,161 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@9984275{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-24T04:21:35,162 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@445a8045{HTTP/1.1, (http/1.1)}{0.0.0.0:39651} 2024-11-24T04:21:35,162 INFO [Time-limited test {}] server.Server(415): Started @7970ms 2024-11-24T04:21:35,166 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T04:21:35,173 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T04:21:35,176 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2024-11-24T04:21:35,182 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-24T04:21:35,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36347 2024-11-24T04:21:35,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36347 2024-11-24T04:21:35,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36347 2024-11-24T04:21:35,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36347 2024-11-24T04:21:35,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36347 2024-11-24T04:21:35,200 DEBUG [M:0;e57b7116ff5e:46719 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e57b7116ff5e:46719 2024-11-24T04:21:35,203 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-24T04:21:35,205 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@234e862b{HTTP/1.1, (http/1.1)}{0.0.0.0:42713} 2024-11-24T04:21:35,205 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] server.Server(415): Started @8013ms 2024-11-24T04:21:35,205 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /1/backup-masters/e57b7116ff5e,46719,1732422093387 2024-11-24T04:21:35,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-24T04:21:35,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-24T04:21:35,223 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/e57b7116ff5e,46719,1732422093387 2024-11-24T04:21:35,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2024-11-24T04:21:35,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:21:35,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:21:35,253 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on existing znode=/1/master 2024-11-24T04:21:35,254 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/e57b7116ff5e,46719,1732422093387 from backup master directory 2024-11-24T04:21:35,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/e57b7116ff5e,46719,1732422093387 2024-11-24T04:21:35,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-24T04:21:35,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-24T04:21:35,263 WARN [master/e57b7116ff5e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T04:21:35,264 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e57b7116ff5e,46719,1732422093387 2024-11-24T04:21:35,266 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T04:21:35,267 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T04:21:35,324 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase.id] with ID: b1b0ce8b-4af7-45ef-973c-2ebbd06519bb 2024-11-24T04:21:35,324 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.tmp/hbase.id 2024-11-24T04:21:35,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741826_1002 (size=42) 2024-11-24T04:21:35,737 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.tmp/hbase.id]:[hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase.id] 2024-11-24T04:21:35,788 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:21:35,794 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T04:21:35,819 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-24T04:21:35,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:21:35,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:21:35,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741827_1003 (size=196) 2024-11-24T04:21:36,320 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:21:36,323 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T04:21:36,331 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-24T04:21:36,334 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-24T04:21:36,346 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.doInit(RegionGroupingProvider.java:156) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.init(AbstractWALProvider.java:126) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:263) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:21:36,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741828_1004 (size=1189) 2024-11-24T04:21:36,402 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store 2024-11-24T04:21:36,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741829_1005 (size=34) 2024-11-24T04:21:36,828 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T04:21:36,832 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:36,834 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T04:21:36,834 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:21:36,834 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:21:36,836 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T04:21:36,836 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:21:36,836 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:21:36,838 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732422096833Disabling compacts and flushes for region at 1732422096833Disabling writes for close at 1732422096836 (+3 ms)Writing region close event to WAL at 1732422096836Closed at 1732422096836 2024-11-24T04:21:36,840 WARN [master/e57b7116ff5e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/.initializing 2024-11-24T04:21:36,840 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/WALs/e57b7116ff5e,46719,1732422093387 2024-11-24T04:21:36,845 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:21:36,850 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor e57b7116ff5e,46719,1732422093387.regiongroup-0 2024-11-24T04:21:36,867 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e57b7116ff5e%2C46719%2C1732422093387.e57b7116ff5e%2C46719%2C1732422093387.regiongroup-0, suffix=, logDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/WALs/e57b7116ff5e,46719,1732422093387, archiveDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/oldWALs, maxLogs=10 2024-11-24T04:21:36,900 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/WALs/e57b7116ff5e,46719,1732422093387/e57b7116ff5e%2C46719%2C1732422093387.e57b7116ff5e%2C46719%2C1732422093387.regiongroup-0.1732422096873, exclude list is [], retry=0 2024-11-24T04:21:36,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:21:36,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T04:21:36,966 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/WALs/e57b7116ff5e,46719,1732422093387/e57b7116ff5e%2C46719%2C1732422093387.e57b7116ff5e%2C46719%2C1732422093387.regiongroup-0.1732422096873 2024-11-24T04:21:36,967 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:21:36,968 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:21:36,968 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:36,973 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:21:36,974 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:21:37,026 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:21:37,053 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T04:21:37,057 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:37,060 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:21:37,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:21:37,065 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T04:21:37,065 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:37,066 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:21:37,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:21:37,070 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T04:21:37,070 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:37,072 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:21:37,072 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:21:37,076 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T04:21:37,076 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:37,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:21:37,078 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:21:37,082 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:21:37,083 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:21:37,090 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:21:37,091 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:21:37,094 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T04:21:37,099 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:21:37,104 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:21:37,106 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9981338080, jitterRate=-0.0704154521226883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T04:21:37,114 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732422096994Initializing all the Stores at 1732422096997 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422096997Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422096999 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422096999Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422097000 (+1 ms)Cleaning up temporary data from old regions at 1732422097091 (+91 ms)Region opened successfully at 1732422097114 (+23 ms) 2024-11-24T04:21:37,116 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T04:21:37,176 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@542cfd9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=e57b7116ff5e/172.17.0.3:0 2024-11-24T04:21:37,223 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T04:21:37,239 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T04:21:37,239 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T04:21:37,242 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T04:21:37,244 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T04:21:37,249 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-24T04:21:37,249 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T04:21:37,276 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T04:21:37,285 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2024-11-24T04:21:37,346 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/balancer already deleted, retry=false 2024-11-24T04:21:37,350 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T04:21:37,352 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2024-11-24T04:21:37,356 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/normalizer already deleted, retry=false 2024-11-24T04:21:37,359 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T04:21:37,364 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2024-11-24T04:21:37,376 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/split already deleted, retry=false 2024-11-24T04:21:37,379 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2024-11-24T04:21:37,388 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/merge already deleted, retry=false 2024-11-24T04:21:37,409 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T04:21:37,419 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/snapshot-cleanup already deleted, retry=false 2024-11-24T04:21:37,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-24T04:21:37,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-24T04:21:37,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:21:37,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:21:37,434 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e57b7116ff5e,46719,1732422093387, sessionid=0x1016b1342860000, setting cluster-up flag (Was=false) 2024-11-24T04:21:37,440 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2024-11-24T04:21:37,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:21:37,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:21:37,493 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2024-11-24T04:21:37,495 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e57b7116ff5e,46719,1732422093387 2024-11-24T04:21:37,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:21:37,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:21:37,546 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2024-11-24T04:21:37,548 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e57b7116ff5e,46719,1732422093387 2024-11-24T04:21:37,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:21:37,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:21:37,599 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2024-11-24T04:21:37,601 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e57b7116ff5e,46719,1732422093387 2024-11-24T04:21:37,609 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T04:21:37,686 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T04:21:37,691 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(746): ClusterId : b1b0ce8b-4af7-45ef-973c-2ebbd06519bb 2024-11-24T04:21:37,692 INFO [RS:0;e57b7116ff5e:36347 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2024-11-24T04:21:37,694 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T04:21:37,698 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T04:21:37,704 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T04:21:37,706 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T04:21:37,706 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2024-11-24T04:21:37,711 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e57b7116ff5e,46719,1732422093387 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T04:21:37,716 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2024-11-24T04:21:37,716 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T04:21:37,718 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e57b7116ff5e:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:21:37,718 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e57b7116ff5e:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:21:37,719 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e57b7116ff5e:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:21:37,719 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e57b7116ff5e:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:21:37,719 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e57b7116ff5e:0, corePoolSize=10, maxPoolSize=10 2024-11-24T04:21:37,719 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e57b7116ff5e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:21:37,719 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e57b7116ff5e:0, corePoolSize=2, maxPoolSize=2 2024-11-24T04:21:37,720 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e57b7116ff5e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:21:37,721 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732422127721 2024-11-24T04:21:37,723 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T04:21:37,724 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T04:21:37,725 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T04:21:37,726 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T04:21:37,726 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T04:21:37,727 DEBUG [RS:0;e57b7116ff5e:36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7ec79e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=e57b7116ff5e/172.17.0.3:0 2024-11-24T04:21:37,729 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T04:21:37,729 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2024-11-24T04:21:37,729 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T04:21:37,730 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T04:21:37,730 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T04:21:37,732 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:37,731 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:37,732 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T04:21:37,736 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T04:21:37,737 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T04:21:37,738 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T04:21:37,743 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e57b7116ff5e:36347 2024-11-24T04:21:37,747 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T04:21:37,747 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T04:21:37,747 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T04:21:37,749 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(2659): reportForDuty to master=e57b7116ff5e,46719,1732422093387 with port=36347, startcode=1732422095008 2024-11-24T04:21:37,755 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e3e6e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:21:37,755 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:21:37,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741831_1007 (size=1321) 2024-11-24T04:21:37,758 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T04:21:37,759 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:21:37,764 DEBUG [RS:0;e57b7116ff5e:36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T04:21:37,764 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:21:37,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741832_1008 (size=32) 2024-11-24T04:21:37,793 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:21:37,799 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41621, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T04:21:37,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:21:37,805 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46719 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T04:21:37,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:21:37,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25b92564, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:21:37,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:21:37,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:21:37,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:21:37,815 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:21:37,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6121cb67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:21:37,818 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-24T04:21:37,824 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T04:21:37,825 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T04:21:37,828 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T04:21:37,828 WARN [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T04:21:37,828 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e57b7116ff5e:0:becomeActiveMaster-HFileCleaner.large.0-1732422097826,5,FailOnTimeoutGroup] 2024-11-24T04:21:37,829 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e57b7116ff5e:0:becomeActiveMaster-HFileCleaner.small.0-1732422097828,5,FailOnTimeoutGroup] 2024-11-24T04:21:37,829 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:37,829 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T04:21:37,831 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:37,831 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:37,929 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(2659): reportForDuty to master=e57b7116ff5e,46719,1732422093387 with port=36347, startcode=1732422095008 2024-11-24T04:21:37,932 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46719 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:37,934 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46719 {}] master.ServerManager(517): Registering regionserver=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:37,944 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:21:37,944 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45347 2024-11-24T04:21:37,944 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=36905 2024-11-24T04:21:37,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-24T04:21:37,988 DEBUG [RS:0;e57b7116ff5e:36347 {}] zookeeper.ZKUtil(111): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on existing znode=/1/rs/e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:37,988 WARN [RS:0;e57b7116ff5e:36347 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T04:21:37,988 INFO [RS:0;e57b7116ff5e:36347 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-24T04:21:37,988 INFO [RS:0;e57b7116ff5e:36347 {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-24T04:21:37,989 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:37,991 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e57b7116ff5e,36347,1732422095008] 2024-11-24T04:21:38,019 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T04:21:38,038 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T04:21:38,045 INFO [RS:0;e57b7116ff5e:36347 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:21:38,045 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,046 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T04:21:38,053 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T04:21:38,055 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,055 DEBUG [RS:0;e57b7116ff5e:36347 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e57b7116ff5e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:21:38,055 DEBUG [RS:0;e57b7116ff5e:36347 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e57b7116ff5e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:21:38,056 DEBUG [RS:0;e57b7116ff5e:36347 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e57b7116ff5e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:21:38,056 DEBUG [RS:0;e57b7116ff5e:36347 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e57b7116ff5e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:21:38,056 DEBUG [RS:0;e57b7116ff5e:36347 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e57b7116ff5e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:21:38,056 DEBUG [RS:0;e57b7116ff5e:36347 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e57b7116ff5e:0, corePoolSize=2, maxPoolSize=2 2024-11-24T04:21:38,056 DEBUG [RS:0;e57b7116ff5e:36347 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e57b7116ff5e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:21:38,056 DEBUG [RS:0;e57b7116ff5e:36347 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e57b7116ff5e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:21:38,057 DEBUG [RS:0;e57b7116ff5e:36347 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e57b7116ff5e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:21:38,057 DEBUG [RS:0;e57b7116ff5e:36347 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e57b7116ff5e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:21:38,057 DEBUG [RS:0;e57b7116ff5e:36347 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e57b7116ff5e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:21:38,057 DEBUG [RS:0;e57b7116ff5e:36347 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e57b7116ff5e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:21:38,057 DEBUG [RS:0;e57b7116ff5e:36347 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:21:38,057 DEBUG [RS:0;e57b7116ff5e:36347 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e57b7116ff5e:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:21:38,058 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,058 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,059 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,059 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,059 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,059 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.ChoreService(168): Chore ScheduledChore name=e57b7116ff5e,36347,1732422095008-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T04:21:38,087 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T04:21:38,089 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.ChoreService(168): Chore ScheduledChore name=e57b7116ff5e,36347,1732422095008-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,090 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,090 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.Replication(171): e57b7116ff5e,36347,1732422095008 started 2024-11-24T04:21:38,116 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,116 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1482): Serving as e57b7116ff5e,36347,1732422095008, RpcServer on e57b7116ff5e/172.17.0.3:36347, sessionid=0x1016b1342860001 2024-11-24T04:21:38,117 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T04:21:38,118 DEBUG [RS:0;e57b7116ff5e:36347 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:38,118 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e57b7116ff5e,36347,1732422095008' 2024-11-24T04:21:38,118 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2024-11-24T04:21:38,119 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2024-11-24T04:21:38,120 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T04:21:38,120 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2024-11-24T04:21:38,120 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e57b7116ff5e,36347,1732422095008' 2024-11-24T04:21:38,120 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T04:21:38,121 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T04:21:38,121 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2024-11-24T04:21:38,121 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2024-11-24T04:21:38,121 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T04:21:38,122 DEBUG [RS:0;e57b7116ff5e:36347 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:38,122 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e57b7116ff5e,36347,1732422095008' 2024-11-24T04:21:38,122 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2024-11-24T04:21:38,122 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2024-11-24T04:21:38,123 DEBUG [RS:0;e57b7116ff5e:36347 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T04:21:38,123 INFO [RS:0;e57b7116ff5e:36347 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T04:21:38,123 INFO [RS:0;e57b7116ff5e:36347 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T04:21:38,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:38,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T04:21:38,183 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T04:21:38,184 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:38,185 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:21:38,185 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T04:21:38,188 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T04:21:38,188 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:38,191 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:21:38,191 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T04:21:38,197 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T04:21:38,198 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:38,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:21:38,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T04:21:38,204 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T04:21:38,204 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:38,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:21:38,206 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T04:21:38,207 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740 2024-11-24T04:21:38,209 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740 2024-11-24T04:21:38,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T04:21:38,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T04:21:38,214 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T04:21:38,217 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T04:21:38,223 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:21:38,224 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9514492480, jitterRate=-0.11389383673667908}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T04:21:38,228 INFO [RS:0;e57b7116ff5e:36347 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:21:38,229 INFO [RS:0;e57b7116ff5e:36347 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e57b7116ff5e,36347,1732422095008.regiongroup-0 2024-11-24T04:21:38,229 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732422098178Initializing all the Stores at 1732422098180 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422098180Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422098180Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422098181 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422098181Cleaning up temporary data from old regions at 1732422098213 (+32 ms)Region opened successfully at 1732422098229 (+16 ms) 2024-11-24T04:21:38,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T04:21:38,230 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T04:21:38,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T04:21:38,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T04:21:38,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T04:21:38,232 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T04:21:38,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732422098229Disabling compacts and flushes for region at 1732422098229Disabling writes for close at 1732422098230 (+1 ms)Writing region close event to WAL at 1732422098231 (+1 ms)Closed at 1732422098232 (+1 ms) 2024-11-24T04:21:38,233 INFO [RS:0;e57b7116ff5e:36347 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0, suffix=, logDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008, archiveDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs, maxLogs=32 2024-11-24T04:21:38,237 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T04:21:38,237 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T04:21:38,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T04:21:38,251 DEBUG [RS:0;e57b7116ff5e:36347 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422098235, exclude list is [], retry=0 2024-11-24T04:21:38,254 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T04:21:38,258 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T04:21:38,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:21:38,263 INFO [RS:0;e57b7116ff5e:36347 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422098235 2024-11-24T04:21:38,264 DEBUG [RS:0;e57b7116ff5e:36347 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:21:38,411 DEBUG [e57b7116ff5e:46719 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T04:21:38,424 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:38,431 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e57b7116ff5e,36347,1732422095008, state=OPENING 2024-11-24T04:21:38,482 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T04:21:38,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:21:38,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:21:38,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-24T04:21:38,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-24T04:21:38,495 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T04:21:38,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:21:38,669 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:21:38,671 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41073, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:21:38,681 INFO [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T04:21:38,681 INFO [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-24T04:21:38,681 INFO [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-24T04:21:38,682 INFO [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:21:38,682 INFO [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T04:21:38,685 INFO [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e57b7116ff5e%2C36347%2C1732422095008.meta, suffix=.meta, logDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008, archiveDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs, maxLogs=32 2024-11-24T04:21:38,699 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422098687.meta, exclude list is [], retry=0 2024-11-24T04:21:38,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:21:38,705 INFO [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422098687.meta 2024-11-24T04:21:38,706 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:21:38,706 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:21:38,710 INFO [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:21:38,711 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T04:21:38,713 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T04:21:38,714 INFO [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T04:21:38,717 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T04:21:38,717 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:38,717 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T04:21:38,717 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T04:21:38,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T04:21:38,722 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T04:21:38,722 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:38,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:21:38,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T04:21:38,724 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T04:21:38,724 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:38,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:21:38,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T04:21:38,727 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T04:21:38,727 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:38,728 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:21:38,728 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T04:21:38,729 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T04:21:38,729 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:38,730 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:21:38,730 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T04:21:38,731 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740 2024-11-24T04:21:38,734 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740 2024-11-24T04:21:38,736 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T04:21:38,736 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T04:21:38,737 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T04:21:38,740 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T04:21:38,741 INFO [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10321286240, jitterRate=-0.03875531256198883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T04:21:38,741 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T04:21:38,745 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732422098718Writing region info on filesystem at 1732422098718Initializing all the Stores at 1732422098720 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422098720Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422098720Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422098720Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422098720Cleaning up temporary data from old regions at 1732422098736 (+16 ms)Running coprocessor post-open hooks at 1732422098741 (+5 ms)Region opened successfully at 1732422098745 (+4 ms) 2024-11-24T04:21:38,751 INFO [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732422098662 2024-11-24T04:21:38,762 DEBUG [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T04:21:38,762 INFO [RS_OPEN_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T04:21:38,764 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:38,766 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e57b7116ff5e,36347,1732422095008, state=OPEN 2024-11-24T04:21:38,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-24T04:21:38,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-24T04:21:38,778 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-24T04:21:38,778 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-24T04:21:38,778 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:38,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T04:21:38,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e57b7116ff5e,36347,1732422095008 in 281 msec 2024-11-24T04:21:38,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T04:21:38,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 542 msec 2024-11-24T04:21:38,794 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T04:21:38,795 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T04:21:38,814 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:21:38,815 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:21:38,833 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:21:38,835 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34021, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:21:38,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2160 sec 2024-11-24T04:21:38,859 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732422098859, completionTime=-1 2024-11-24T04:21:38,862 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-24T04:21:38,863 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T04:21:38,896 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-24T04:21:38,896 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732422158896 2024-11-24T04:21:38,896 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732422218896 2024-11-24T04:21:38,896 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-11-24T04:21:38,898 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e57b7116ff5e,46719,1732422093387-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,898 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e57b7116ff5e,46719,1732422093387-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,898 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e57b7116ff5e,46719,1732422093387-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,899 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e57b7116ff5e:46719, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,900 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,900 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,907 DEBUG [master/e57b7116ff5e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T04:21:38,934 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.670sec 2024-11-24T04:21:38,935 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T04:21:38,936 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T04:21:38,937 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T04:21:38,937 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T04:21:38,938 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T04:21:38,938 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e57b7116ff5e,46719,1732422093387-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T04:21:38,939 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e57b7116ff5e,46719,1732422093387-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T04:21:38,947 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T04:21:38,947 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T04:21:38,948 INFO [master/e57b7116ff5e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e57b7116ff5e,46719,1732422093387-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:21:38,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cc25be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:21:39,000 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T04:21:39,000 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T04:21:39,003 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:21:39,003 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:21:39,009 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:21:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:21:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:21:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51474a70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:21:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:21:39,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:21:39,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:21:39,014 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:21:39,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195abe33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:21:39,016 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:21:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:21:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:21:39,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:21:39,044 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e57b7116ff5e,46719,1732422093387 2024-11-24T04:21:39,044 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2305): Starting mini mapreduce cluster... 2024-11-24T04:21:39,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/test.cache.data in system properties and HBase conf 2024-11-24T04:21:39,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T04:21:39,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir in system properties and HBase conf 2024-11-24T04:21:39,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T04:21:39,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T04:21:39,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T04:21:39,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T04:21:39,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T04:21:39,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T04:21:39,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T04:21:39,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T04:21:39,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T04:21:39,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T04:21:39,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T04:21:39,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T04:21:39,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/nfs.dump.dir in system properties and HBase conf 2024-11-24T04:21:39,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/java.io.tmpdir in system properties and HBase conf 2024-11-24T04:21:39,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T04:21:39,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T04:21:39,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T04:21:39,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741835_1011 (size=592039) 2024-11-24T04:21:39,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741836_1012 (size=1663647) 2024-11-24T04:21:41,147 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:21:41,217 WARN [Thread-251 {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:21:41,448 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:21:41,448 WARN [Thread-251 {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-24T04:21:41,449 INFO [Thread-251 {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:21:41,450 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:21:41,450 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:21:41,451 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T04:21:41,455 INFO [Thread-251 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:21:41,455 INFO [Thread-251 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:21:41,456 INFO [Thread-251 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T04:21:41,456 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@37164ab0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:21:41,457 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:21:41,457 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2860dd4f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-24T04:21:41,464 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@339da2c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:21:41,464 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11a358cb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-24T04:21:41,602 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2024-11-24T04:21:41,602 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2024-11-24T04:21:41,602 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-24T04:21:41,604 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-24T04:21:41,644 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T04:21:41,691 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T04:21:41,990 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2024-11-24T04:21:42,015 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5ca13ded{jobhistory,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/java.io.tmpdir/jetty-localhost-38313-hadoop-yarn-common-3_4_1_jar-_-any-18036861783060290568/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-24T04:21:42,015 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@52227462{cluster,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/java.io.tmpdir/jetty-localhost-40493-hadoop-yarn-common-3_4_1_jar-_-any-14465127005086049750/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-24T04:21:42,016 INFO [Thread-251 {}] server.AbstractConnector(333): Started ServerConnector@7eba726a{HTTP/1.1, (http/1.1)}{localhost:38313} 2024-11-24T04:21:42,016 INFO [Thread-251 {}] server.Server(415): Started @14824ms 2024-11-24T04:21:42,016 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@382d96c0{HTTP/1.1, (http/1.1)}{localhost:40493} 2024-11-24T04:21:42,016 INFO [Time-limited test {}] server.Server(415): Started @14825ms 2024-11-24T04:21:42,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741837_1013 (size=5) 2024-11-24T04:21:43,079 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-24T04:21:43,083 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:21:43,120 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-24T04:21:43,121 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:21:43,128 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:21:43,128 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:21:43,129 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T04:21:43,130 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:21:43,131 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bdc2c31{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:21:43,131 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@722bd21f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-24T04:21:43,179 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-24T04:21:43,179 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-24T04:21:43,179 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-24T04:21:43,180 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-24T04:21:43,202 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T04:21:43,209 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T04:21:43,330 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T04:21:43,343 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@39275f7{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/java.io.tmpdir/jetty-localhost-43035-hadoop-yarn-common-3_4_1_jar-_-any-3966791704597337075/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-24T04:21:43,344 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@159887a6{HTTP/1.1, (http/1.1)}{localhost:43035} 2024-11-24T04:21:43,344 INFO [Time-limited test {}] server.Server(415): Started @16152ms 2024-11-24T04:21:43,757 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-24T04:21:43,760 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:21:43,771 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-24T04:21:43,772 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:21:43,777 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:21:43,777 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:21:43,777 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T04:21:43,777 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:21:43,778 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d51f859{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:21:43,778 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@436a6a5d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-24T04:21:43,830 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-24T04:21:43,831 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-24T04:21:43,831 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-24T04:21:43,831 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-24T04:21:43,841 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T04:21:43,847 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T04:21:43,937 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T04:21:43,941 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@43aedd1d{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/java.io.tmpdir/jetty-localhost-37549-hadoop-yarn-common-3_4_1_jar-_-any-555150732717261668/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-24T04:21:43,942 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4c6d042f{HTTP/1.1, (http/1.1)}{localhost:37549} 2024-11-24T04:21:43,942 INFO [Time-limited test {}] server.Server(415): Started @16750ms 2024-11-24T04:21:43,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2341): Mini mapreduce cluster started 2024-11-24T04:21:43,966 INFO [Time-limited test {}] backup.TestBackupBase(336): ROOTDIR hdfs://localhost:45347/backupUT 2024-11-24T04:21:43,969 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:21:43,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:21:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@646850f5 2024-11-24T04:21:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:21:43,981 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:21:43,984 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns1'} 2024-11-24T04:21:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns1 2024-11-24T04:21:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T04:21:44,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns1 in 26 msec 2024-11-24T04:21:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T04:21:44,112 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns1 completed 2024-11-24T04:21:44,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns2'} 2024-11-24T04:21:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns2 2024-11-24T04:21:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-24T04:21:44,132 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns2 in 15 msec 2024-11-24T04:21:44,227 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-24T04:21:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-24T04:21:44,232 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns2 completed 2024-11-24T04:21:44,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns3'} 2024-11-24T04:21:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=6, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns3 2024-11-24T04:21:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-24T04:21:44,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns3 in 14 msec 2024-11-24T04:21:44,295 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T04:21:44,295 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T04:21:44,297 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:21:44,297 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2024-11-24T04:21:44,298 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T04:21:44,298 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T04:21:44,298 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T04:21:44,298 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T04:21:44,302 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T04:21:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-24T04:21:44,342 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns3 completed 2024-11-24T04:21:44,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns4'} 2024-11-24T04:21:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns4 2024-11-24T04:21:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T04:21:44,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns4 in 14 msec 2024-11-24T04:21:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T04:21:44,451 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns4 completed 2024-11-24T04:21:44,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test-1732422103966', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:21:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test-1732422103966 2024-11-24T04:21:44,469 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test-1732422103966 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T04:21:44,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test-1732422103966" procId is: 8 2024-11-24T04:21:44,471 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-24T04:21:44,473 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test-1732422103966 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T04:21:44,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741838_1014 (size=367) 2024-11-24T04:21:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-24T04:21:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-24T04:21:44,887 INFO [RegionOpenAndInit-test-1732422103966-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2c375e169f1afc2d9cce55928a15d594, NAME => 'test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1732422103966', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:21:44,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741839_1015 (size=53) 2024-11-24T04:21:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-24T04:21:45,298 DEBUG [RegionOpenAndInit-test-1732422103966-pool-0 {}] regionserver.HRegion(898): Instantiated test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:45,298 DEBUG [RegionOpenAndInit-test-1732422103966-pool-0 {}] regionserver.HRegion(1722): Closing 2c375e169f1afc2d9cce55928a15d594, disabling compactions & flushes 2024-11-24T04:21:45,298 INFO [RegionOpenAndInit-test-1732422103966-pool-0 {}] regionserver.HRegion(1755): Closing region test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:21:45,298 DEBUG [RegionOpenAndInit-test-1732422103966-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:21:45,298 DEBUG [RegionOpenAndInit-test-1732422103966-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. after waiting 0 ms 2024-11-24T04:21:45,298 DEBUG [RegionOpenAndInit-test-1732422103966-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:21:45,298 INFO [RegionOpenAndInit-test-1732422103966-pool-0 {}] regionserver.HRegion(1973): Closed test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:21:45,298 DEBUG [RegionOpenAndInit-test-1732422103966-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2c375e169f1afc2d9cce55928a15d594: Waiting for close lock at 1732422105298Disabling compacts and flushes for region at 1732422105298Disabling writes for close at 1732422105298Writing region close event to WAL at 1732422105298Closed at 1732422105298 2024-11-24T04:21:45,300 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test-1732422103966 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T04:21:45,305 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732422105300"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422105300"}]},"ts":"1732422105300"} 2024-11-24T04:21:45,309 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T04:21:45,311 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test-1732422103966 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T04:21:45,314 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1732422103966","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422105311"}]},"ts":"1732422105311"} 2024-11-24T04:21:45,318 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1732422103966, state=ENABLING in hbase:meta 2024-11-24T04:21:45,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, ASSIGN}] 2024-11-24T04:21:45,322 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, ASSIGN 2024-11-24T04:21:45,324 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, ASSIGN; state=OFFLINE, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=false 2024-11-24T04:21:45,475 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=2c375e169f1afc2d9cce55928a15d594, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:45,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, ASSIGN because future has completed 2024-11-24T04:21:45,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c375e169f1afc2d9cce55928a15d594, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:21:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-24T04:21:45,655 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(132): Open test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:21:45,655 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:21:45,655 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] monitor.StreamSlowMonitor(122): New stream slow monitor e57b7116ff5e,36347,1732422095008.regiongroup-1 2024-11-24T04:21:45,658 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1, suffix=, logDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008, archiveDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs, maxLogs=32 2024-11-24T04:21:45,673 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422105660, exclude list is [], retry=0 2024-11-24T04:21:45,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:21:45,680 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422105660 2024-11-24T04:21:45,681 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:21:45,681 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7752): Opening region: {ENCODED => 2c375e169f1afc2d9cce55928a15d594, NAME => 'test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:21:45,681 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:21:45,681 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732422103966 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:45,682 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(898): Instantiated test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:45,682 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7794): checking encryption for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:45,682 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7797): checking classloading for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:45,685 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:45,688 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c375e169f1afc2d9cce55928a15d594 columnFamilyName f 2024-11-24T04:21:45,688 DEBUG [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:45,690 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.HStore(327): Store=2c375e169f1afc2d9cce55928a15d594/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:21:45,690 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1038): replaying wal for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:45,691 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:45,692 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:45,693 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1048): stopping wal replay for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:45,693 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1060): Cleaning up temporary data for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:45,696 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1093): writing seq id for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:45,700 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:21:45,701 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1114): Opened 2c375e169f1afc2d9cce55928a15d594; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11743324480, jitterRate=0.09368231892585754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:21:45,702 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:45,703 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1006): Region open journal for 2c375e169f1afc2d9cce55928a15d594: Running coprocessor pre-open hook at 1732422105682Writing region info on filesystem at 1732422105682Initializing all the Stores at 1732422105684 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422105684Cleaning up temporary data from old regions at 1732422105693 (+9 ms)Running coprocessor post-open hooks at 1732422105702 (+9 ms)Region opened successfully at 1732422105703 (+1 ms) 2024-11-24T04:21:45,706 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594., pid=10, masterSystemTime=1732422105635 2024-11-24T04:21:45,710 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:21:45,710 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(153): Opened test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:21:45,711 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=2c375e169f1afc2d9cce55928a15d594, regionState=OPEN, openSeqNum=2, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:45,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c375e169f1afc2d9cce55928a15d594, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:21:45,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-24T04:21:45,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 2c375e169f1afc2d9cce55928a15d594, server=e57b7116ff5e,36347,1732422095008 in 238 msec 2024-11-24T04:21:45,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-24T04:21:45,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, ASSIGN in 405 msec 2024-11-24T04:21:45,733 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test-1732422103966 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T04:21:45,733 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1732422103966","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422105733"}]},"ts":"1732422105733"} 2024-11-24T04:21:45,736 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1732422103966, state=ENABLED in hbase:meta 2024-11-24T04:21:45,738 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test-1732422103966 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T04:21:45,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; CreateTableProcedure table=test-1732422103966 in 1.2770 sec 2024-11-24T04:21:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-24T04:21:46,632 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test-1732422103966 completed 2024-11-24T04:21:46,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c4e82b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:21:46,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:21:46,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:21:46,634 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:21:46,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:21:46,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:21:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2485c6aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:21:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:21:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:21:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:21:46,638 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:21:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34cc121d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:21:46,642 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:21:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:21:46,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:21:46,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:21:46,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732422103966', row='row0', locateType=CURRENT is [region=test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:21:46,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HRegion(8528): writing data to region test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. with WAL disabled. Data may be lost in the event of a crash. 2024-11-24T04:21:46,850 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns2:test-17324221039661', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:21:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:test-17324221039661 2024-11-24T04:21:46,854 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17324221039661 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T04:21:46,854 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:46,854 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns2" qualifier: "test-17324221039661" procId is: 11 2024-11-24T04:21:46,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T04:21:46,856 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:test-17324221039661 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T04:21:46,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741841_1017 (size=364) 2024-11-24T04:21:46,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T04:21:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T04:21:47,268 INFO [RegionOpenAndInit-ns2:test-17324221039661-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d3f34500755154053262a8d9790ac0a1, NAME => 'ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17324221039661', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:21:47,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741842_1018 (size=50) 2024-11-24T04:21:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T04:21:47,679 DEBUG [RegionOpenAndInit-ns2:test-17324221039661-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:47,679 DEBUG [RegionOpenAndInit-ns2:test-17324221039661-pool-0 {}] regionserver.HRegion(1722): Closing d3f34500755154053262a8d9790ac0a1, disabling compactions & flushes 2024-11-24T04:21:47,679 INFO [RegionOpenAndInit-ns2:test-17324221039661-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:21:47,679 DEBUG [RegionOpenAndInit-ns2:test-17324221039661-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:21:47,679 DEBUG [RegionOpenAndInit-ns2:test-17324221039661-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. after waiting 0 ms 2024-11-24T04:21:47,679 DEBUG [RegionOpenAndInit-ns2:test-17324221039661-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:21:47,679 INFO [RegionOpenAndInit-ns2:test-17324221039661-pool-0 {}] regionserver.HRegion(1973): Closed ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:21:47,679 DEBUG [RegionOpenAndInit-ns2:test-17324221039661-pool-0 {}] regionserver.HRegion(1676): Region close journal for d3f34500755154053262a8d9790ac0a1: Waiting for close lock at 1732422107679Disabling compacts and flushes for region at 1732422107679Disabling writes for close at 1732422107679Writing region close event to WAL at 1732422107679Closed at 1732422107679 2024-11-24T04:21:47,681 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:test-17324221039661 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T04:21:47,682 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732422107681"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422107681"}]},"ts":"1732422107681"} 2024-11-24T04:21:47,685 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T04:21:47,687 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:test-17324221039661 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T04:21:47,687 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17324221039661","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422107687"}]},"ts":"1732422107687"} 2024-11-24T04:21:47,690 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17324221039661, state=ENABLING in hbase:meta 2024-11-24T04:21:47,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17324221039661, region=d3f34500755154053262a8d9790ac0a1, ASSIGN}] 2024-11-24T04:21:47,692 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17324221039661, region=d3f34500755154053262a8d9790ac0a1, ASSIGN 2024-11-24T04:21:47,694 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:test-17324221039661, region=d3f34500755154053262a8d9790ac0a1, ASSIGN; state=OFFLINE, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=false 2024-11-24T04:21:47,845 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=d3f34500755154053262a8d9790ac0a1, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:47,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:test-17324221039661, region=d3f34500755154053262a8d9790ac0a1, ASSIGN because future has completed 2024-11-24T04:21:47,849 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3f34500755154053262a8d9790ac0a1, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:21:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T04:21:48,025 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:21:48,026 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => d3f34500755154053262a8d9790ac0a1, NAME => 'ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:21:48,026 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:21:48,027 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17324221039661 d3f34500755154053262a8d9790ac0a1 2024-11-24T04:21:48,027 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:48,027 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for d3f34500755154053262a8d9790ac0a1 2024-11-24T04:21:48,027 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for d3f34500755154053262a8d9790ac0a1 2024-11-24T04:21:48,029 INFO [StoreOpener-d3f34500755154053262a8d9790ac0a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d3f34500755154053262a8d9790ac0a1 2024-11-24T04:21:48,032 INFO [StoreOpener-d3f34500755154053262a8d9790ac0a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3f34500755154053262a8d9790ac0a1 columnFamilyName f 2024-11-24T04:21:48,032 DEBUG [StoreOpener-d3f34500755154053262a8d9790ac0a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:48,033 INFO [StoreOpener-d3f34500755154053262a8d9790ac0a1-1 {}] regionserver.HStore(327): Store=d3f34500755154053262a8d9790ac0a1/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:21:48,033 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for d3f34500755154053262a8d9790ac0a1 2024-11-24T04:21:48,035 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1 2024-11-24T04:21:48,035 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1 2024-11-24T04:21:48,036 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for d3f34500755154053262a8d9790ac0a1 2024-11-24T04:21:48,036 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for d3f34500755154053262a8d9790ac0a1 2024-11-24T04:21:48,040 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for d3f34500755154053262a8d9790ac0a1 2024-11-24T04:21:48,044 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:21:48,045 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened d3f34500755154053262a8d9790ac0a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10303544000, jitterRate=-0.0404076874256134}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:21:48,045 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3f34500755154053262a8d9790ac0a1 2024-11-24T04:21:48,047 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for d3f34500755154053262a8d9790ac0a1: Running coprocessor pre-open hook at 1732422108027Writing region info on filesystem at 1732422108027Initializing all the Stores at 1732422108029 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422108029Cleaning up temporary data from old regions at 1732422108036 (+7 ms)Running coprocessor post-open hooks at 1732422108045 (+9 ms)Region opened successfully at 1732422108046 (+1 ms) 2024-11-24T04:21:48,048 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1., pid=13, masterSystemTime=1732422108003 2024-11-24T04:21:48,051 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:21:48,051 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:21:48,052 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=d3f34500755154053262a8d9790ac0a1, regionState=OPEN, openSeqNum=2, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:48,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3f34500755154053262a8d9790ac0a1, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:21:48,061 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-11-24T04:21:48,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure d3f34500755154053262a8d9790ac0a1, server=e57b7116ff5e,36347,1732422095008 in 209 msec 2024-11-24T04:21:48,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-24T04:21:48,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:test-17324221039661, region=d3f34500755154053262a8d9790ac0a1, ASSIGN in 372 msec 2024-11-24T04:21:48,068 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:test-17324221039661 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T04:21:48,069 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17324221039661","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422108069"}]},"ts":"1732422108069"} 2024-11-24T04:21:48,072 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17324221039661, state=ENABLED in hbase:meta 2024-11-24T04:21:48,074 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17324221039661 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T04:21:48,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:test-17324221039661 in 1.2230 sec 2024-11-24T04:21:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T04:21:49,001 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:test-17324221039661 completed 2024-11-24T04:21:49,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17324221039661', row='row0', locateType=CURRENT is [region=ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:21:49,006 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HRegion(8528): writing data to region ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. with WAL disabled. Data may be lost in the event of a crash. 2024-11-24T04:21:49,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns3:test-17324221039662', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:21:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns3:test-17324221039662 2024-11-24T04:21:49,158 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17324221039662 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T04:21:49,158 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:49,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns3" qualifier: "test-17324221039662" procId is: 14 2024-11-24T04:21:49,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-24T04:21:49,160 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns3:test-17324221039662 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T04:21:49,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741843_1019 (size=390) 2024-11-24T04:21:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-24T04:21:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-24T04:21:49,573 INFO [RegionOpenAndInit-ns3:test-17324221039662-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a35f3acd735a5c01c6819f7a642d051d, NAME => 'ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17324221039662', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:21:49,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741844_1020 (size=50) 2024-11-24T04:21:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-24T04:21:49,984 DEBUG [RegionOpenAndInit-ns3:test-17324221039662-pool-0 {}] regionserver.HRegion(898): Instantiated ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:49,984 DEBUG [RegionOpenAndInit-ns3:test-17324221039662-pool-0 {}] regionserver.HRegion(1722): Closing a35f3acd735a5c01c6819f7a642d051d, disabling compactions & flushes 2024-11-24T04:21:49,985 INFO [RegionOpenAndInit-ns3:test-17324221039662-pool-0 {}] regionserver.HRegion(1755): Closing region ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. 2024-11-24T04:21:49,985 DEBUG [RegionOpenAndInit-ns3:test-17324221039662-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. 2024-11-24T04:21:49,985 DEBUG [RegionOpenAndInit-ns3:test-17324221039662-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. after waiting 0 ms 2024-11-24T04:21:49,985 DEBUG [RegionOpenAndInit-ns3:test-17324221039662-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. 2024-11-24T04:21:49,985 INFO [RegionOpenAndInit-ns3:test-17324221039662-pool-0 {}] regionserver.HRegion(1973): Closed ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. 2024-11-24T04:21:49,985 DEBUG [RegionOpenAndInit-ns3:test-17324221039662-pool-0 {}] regionserver.HRegion(1676): Region close journal for a35f3acd735a5c01c6819f7a642d051d: Waiting for close lock at 1732422109984Disabling compacts and flushes for region at 1732422109984Disabling writes for close at 1732422109985 (+1 ms)Writing region close event to WAL at 1732422109985Closed at 1732422109985 2024-11-24T04:21:49,987 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns3:test-17324221039662 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T04:21:49,987 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732422109987"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422109987"}]},"ts":"1732422109987"} 2024-11-24T04:21:49,990 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T04:21:49,991 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns3:test-17324221039662 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T04:21:49,992 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17324221039662","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422109991"}]},"ts":"1732422109991"} 2024-11-24T04:21:49,994 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17324221039662, state=ENABLING in hbase:meta 2024-11-24T04:21:49,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17324221039662, region=a35f3acd735a5c01c6819f7a642d051d, ASSIGN}] 2024-11-24T04:21:49,997 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17324221039662, region=a35f3acd735a5c01c6819f7a642d051d, ASSIGN 2024-11-24T04:21:49,999 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns3:test-17324221039662, region=a35f3acd735a5c01c6819f7a642d051d, ASSIGN; state=OFFLINE, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=false 2024-11-24T04:21:50,150 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=a35f3acd735a5c01c6819f7a642d051d, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:50,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns3:test-17324221039662, region=a35f3acd735a5c01c6819f7a642d051d, ASSIGN because future has completed 2024-11-24T04:21:50,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure a35f3acd735a5c01c6819f7a642d051d, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:21:50,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-24T04:21:50,305 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17324221039661' 2024-11-24T04:21:50,306 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1732422103966' 2024-11-24T04:21:50,314 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. 2024-11-24T04:21:50,314 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => a35f3acd735a5c01c6819f7a642d051d, NAME => 'ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:21:50,315 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:21:50,315 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17324221039662 a35f3acd735a5c01c6819f7a642d051d 2024-11-24T04:21:50,315 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:50,315 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for a35f3acd735a5c01c6819f7a642d051d 2024-11-24T04:21:50,315 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for a35f3acd735a5c01c6819f7a642d051d 2024-11-24T04:21:50,317 INFO [StoreOpener-a35f3acd735a5c01c6819f7a642d051d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a35f3acd735a5c01c6819f7a642d051d 2024-11-24T04:21:50,320 INFO [StoreOpener-a35f3acd735a5c01c6819f7a642d051d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a35f3acd735a5c01c6819f7a642d051d columnFamilyName f 2024-11-24T04:21:50,320 DEBUG [StoreOpener-a35f3acd735a5c01c6819f7a642d051d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:50,321 INFO [StoreOpener-a35f3acd735a5c01c6819f7a642d051d-1 {}] regionserver.HStore(327): Store=a35f3acd735a5c01c6819f7a642d051d/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:21:50,321 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for a35f3acd735a5c01c6819f7a642d051d 2024-11-24T04:21:50,322 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns3/test-17324221039662/a35f3acd735a5c01c6819f7a642d051d 2024-11-24T04:21:50,323 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns3/test-17324221039662/a35f3acd735a5c01c6819f7a642d051d 2024-11-24T04:21:50,324 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for a35f3acd735a5c01c6819f7a642d051d 2024-11-24T04:21:50,324 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for a35f3acd735a5c01c6819f7a642d051d 2024-11-24T04:21:50,327 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for a35f3acd735a5c01c6819f7a642d051d 2024-11-24T04:21:50,331 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns3/test-17324221039662/a35f3acd735a5c01c6819f7a642d051d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:21:50,332 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened a35f3acd735a5c01c6819f7a642d051d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11323396160, jitterRate=0.05457344651222229}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:21:50,332 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a35f3acd735a5c01c6819f7a642d051d 2024-11-24T04:21:50,333 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for a35f3acd735a5c01c6819f7a642d051d: Running coprocessor pre-open hook at 1732422110315Writing region info on filesystem at 1732422110315Initializing all the Stores at 1732422110317 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422110317Cleaning up temporary data from old regions at 1732422110324 (+7 ms)Running coprocessor post-open hooks at 1732422110332 (+8 ms)Region opened successfully at 1732422110333 (+1 ms) 2024-11-24T04:21:50,335 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d., pid=16, masterSystemTime=1732422110308 2024-11-24T04:21:50,338 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. 2024-11-24T04:21:50,338 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. 2024-11-24T04:21:50,339 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=a35f3acd735a5c01c6819f7a642d051d, regionState=OPEN, openSeqNum=2, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:50,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure a35f3acd735a5c01c6819f7a642d051d, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:21:50,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-24T04:21:50,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure a35f3acd735a5c01c6819f7a642d051d, server=e57b7116ff5e,36347,1732422095008 in 191 msec 2024-11-24T04:21:50,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-24T04:21:50,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns3:test-17324221039662, region=a35f3acd735a5c01c6819f7a642d051d, ASSIGN in 355 msec 2024-11-24T04:21:50,357 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns3:test-17324221039662 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T04:21:50,357 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17324221039662","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422110357"}]},"ts":"1732422110357"} 2024-11-24T04:21:50,361 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17324221039662, state=ENABLED in hbase:meta 2024-11-24T04:21:50,362 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17324221039662 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T04:21:50,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns3:test-17324221039662 in 1.2080 sec 2024-11-24T04:21:51,186 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-24T04:21:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-24T04:21:51,311 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns3:test-17324221039662 completed 2024-11-24T04:21:51,312 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns3:test-17324221039662 get assigned. Timeout = 60000ms 2024-11-24T04:21:51,313 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T04:21:51,319 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns3:test-17324221039662 assigned to meta. Checking AM states. 2024-11-24T04:21:51,319 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T04:21:51,320 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns3:test-17324221039662 assigned. 2024-11-24T04:21:51,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns4:test-17324221039663', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:21:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns4:test-17324221039663 2024-11-24T04:21:51,326 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17324221039663 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T04:21:51,326 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:51,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns4" qualifier: "test-17324221039663" procId is: 17 2024-11-24T04:21:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T04:21:51,360 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns4:test-17324221039663 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T04:21:51,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741845_1021 (size=390) 2024-11-24T04:21:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T04:21:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T04:21:51,772 INFO [RegionOpenAndInit-ns4:test-17324221039663-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a75e6c6417a138961442a072386b6bcc, NAME => 'ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17324221039663', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:21:51,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741846_1022 (size=50) 2024-11-24T04:21:51,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T04:21:52,182 DEBUG [RegionOpenAndInit-ns4:test-17324221039663-pool-0 {}] regionserver.HRegion(898): Instantiated ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:52,182 DEBUG [RegionOpenAndInit-ns4:test-17324221039663-pool-0 {}] regionserver.HRegion(1722): Closing a75e6c6417a138961442a072386b6bcc, disabling compactions & flushes 2024-11-24T04:21:52,182 INFO [RegionOpenAndInit-ns4:test-17324221039663-pool-0 {}] regionserver.HRegion(1755): Closing region ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. 2024-11-24T04:21:52,182 DEBUG [RegionOpenAndInit-ns4:test-17324221039663-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. 2024-11-24T04:21:52,182 DEBUG [RegionOpenAndInit-ns4:test-17324221039663-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. after waiting 0 ms 2024-11-24T04:21:52,183 DEBUG [RegionOpenAndInit-ns4:test-17324221039663-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. 2024-11-24T04:21:52,183 INFO [RegionOpenAndInit-ns4:test-17324221039663-pool-0 {}] regionserver.HRegion(1973): Closed ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. 2024-11-24T04:21:52,183 DEBUG [RegionOpenAndInit-ns4:test-17324221039663-pool-0 {}] regionserver.HRegion(1676): Region close journal for a75e6c6417a138961442a072386b6bcc: Waiting for close lock at 1732422112182Disabling compacts and flushes for region at 1732422112182Disabling writes for close at 1732422112182Writing region close event to WAL at 1732422112183 (+1 ms)Closed at 1732422112183 2024-11-24T04:21:52,185 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns4:test-17324221039663 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T04:21:52,185 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732422112185"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422112185"}]},"ts":"1732422112185"} 2024-11-24T04:21:52,188 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T04:21:52,190 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns4:test-17324221039663 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T04:21:52,190 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17324221039663","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422112190"}]},"ts":"1732422112190"} 2024-11-24T04:21:52,193 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17324221039663, state=ENABLING in hbase:meta 2024-11-24T04:21:52,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17324221039663, region=a75e6c6417a138961442a072386b6bcc, ASSIGN}] 2024-11-24T04:21:52,196 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17324221039663, region=a75e6c6417a138961442a072386b6bcc, ASSIGN 2024-11-24T04:21:52,197 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns4:test-17324221039663, region=a75e6c6417a138961442a072386b6bcc, ASSIGN; state=OFFLINE, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=false 2024-11-24T04:21:52,348 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=a75e6c6417a138961442a072386b6bcc, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:52,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns4:test-17324221039663, region=a75e6c6417a138961442a072386b6bcc, ASSIGN because future has completed 2024-11-24T04:21:52,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure a75e6c6417a138961442a072386b6bcc, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:21:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T04:21:52,511 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. 2024-11-24T04:21:52,511 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => a75e6c6417a138961442a072386b6bcc, NAME => 'ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:21:52,512 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:21:52,512 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17324221039663 a75e6c6417a138961442a072386b6bcc 2024-11-24T04:21:52,512 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:52,512 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for a75e6c6417a138961442a072386b6bcc 2024-11-24T04:21:52,512 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for a75e6c6417a138961442a072386b6bcc 2024-11-24T04:21:52,515 INFO [StoreOpener-a75e6c6417a138961442a072386b6bcc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a75e6c6417a138961442a072386b6bcc 2024-11-24T04:21:52,518 INFO [StoreOpener-a75e6c6417a138961442a072386b6bcc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a75e6c6417a138961442a072386b6bcc columnFamilyName f 2024-11-24T04:21:52,518 DEBUG [StoreOpener-a75e6c6417a138961442a072386b6bcc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:52,519 INFO [StoreOpener-a75e6c6417a138961442a072386b6bcc-1 {}] regionserver.HStore(327): Store=a75e6c6417a138961442a072386b6bcc/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:21:52,519 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for a75e6c6417a138961442a072386b6bcc 2024-11-24T04:21:52,521 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns4/test-17324221039663/a75e6c6417a138961442a072386b6bcc 2024-11-24T04:21:52,521 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns4/test-17324221039663/a75e6c6417a138961442a072386b6bcc 2024-11-24T04:21:52,522 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for a75e6c6417a138961442a072386b6bcc 2024-11-24T04:21:52,522 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for a75e6c6417a138961442a072386b6bcc 2024-11-24T04:21:52,524 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for a75e6c6417a138961442a072386b6bcc 2024-11-24T04:21:52,527 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns4/test-17324221039663/a75e6c6417a138961442a072386b6bcc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:21:52,528 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened a75e6c6417a138961442a072386b6bcc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10099615200, jitterRate=-0.0594000369310379}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:21:52,528 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a75e6c6417a138961442a072386b6bcc 2024-11-24T04:21:52,528 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for a75e6c6417a138961442a072386b6bcc: Running coprocessor pre-open hook at 1732422112513Writing region info on filesystem at 1732422112513Initializing all the Stores at 1732422112514 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422112514Cleaning up temporary data from old regions at 1732422112522 (+8 ms)Running coprocessor post-open hooks at 1732422112528 (+6 ms)Region opened successfully at 1732422112528 2024-11-24T04:21:52,530 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc., pid=19, masterSystemTime=1732422112506 2024-11-24T04:21:52,533 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. 2024-11-24T04:21:52,533 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. 2024-11-24T04:21:52,534 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=a75e6c6417a138961442a072386b6bcc, regionState=OPEN, openSeqNum=2, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:52,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure a75e6c6417a138961442a072386b6bcc, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:21:52,543 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-24T04:21:52,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure a75e6c6417a138961442a072386b6bcc, server=e57b7116ff5e,36347,1732422095008 in 187 msec 2024-11-24T04:21:52,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-24T04:21:52,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns4:test-17324221039663, region=a75e6c6417a138961442a072386b6bcc, ASSIGN in 349 msec 2024-11-24T04:21:52,547 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns4:test-17324221039663 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T04:21:52,547 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17324221039663","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422112547"}]},"ts":"1732422112547"} 2024-11-24T04:21:52,550 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17324221039663, state=ENABLED in hbase:meta 2024-11-24T04:21:52,552 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17324221039663 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T04:21:52,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns4:test-17324221039663 in 1.2300 sec 2024-11-24T04:21:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T04:21:53,511 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns4:test-17324221039663 completed 2024-11-24T04:21:53,511 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns4:test-17324221039663 get assigned. Timeout = 60000ms 2024-11-24T04:21:53,512 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T04:21:53,517 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns4:test-17324221039663 assigned to meta. Checking AM states. 2024-11-24T04:21:53,517 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T04:21:53,517 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns4:test-17324221039663 assigned. 2024-11-24T04:21:53,518 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:21:53,518 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestBackupBase.createTables(TestBackupBase.java:481) at org.apache.hadoop.hbase.backup.TestBackupBase.setUpHelper(TestBackupBase.java:343) at org.apache.hadoop.hbase.backup.TestBackupBase.setUp(TestBackupBase.java:357) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:21:53,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:21:53,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:21:53,526 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:21:53,555 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=611, OpenFileDescriptor=633, MaxFileDescriptor=1048576, SystemLoadAverage=207, ProcessCount=15, AvailableMemoryMB=7990 2024-11-24T04:21:53,557 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=611 is superior to 500 2024-11-24T04:21:53,558 INFO [Time-limited test {}] backup.TestIncrementalBackup(91): create full backup image for all tables 2024-11-24T04:21:53,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table test-1732422103966 from 'test-1732422103966', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1732422103966', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:21:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=20, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1732422103966 2024-11-24T04:21:53,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741847_1023 (size=908) 2024-11-24T04:21:53,778 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17324221039661 2024-11-24T04:21:53,778 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17324221039661 Metrics about Tables on a single HBase RegionServer 2024-11-24T04:21:53,780 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1732422103966 2024-11-24T04:21:53,780 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1732422103966 Metrics about Tables on a single HBase RegionServer 2024-11-24T04:21:53,781 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17324221039662 2024-11-24T04:21:53,781 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17324221039662 Metrics about Tables on a single HBase RegionServer 2024-11-24T04:21:53,782 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17324221039663 2024-11-24T04:21:53,782 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17324221039663 Metrics about Tables on a single HBase RegionServer 2024-11-24T04:21:53,989 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/.tabledesc/.tableinfo.0000000001.367 2024-11-24T04:21:53,989 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/.tabledesc/.tableinfo.0000000002.908 2024-11-24T04:21:53,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1732422103966}] 2024-11-24T04:21:54,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, REOPEN/MOVE}] 2024-11-24T04:21:54,003 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, REOPEN/MOVE 2024-11-24T04:21:54,005 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=2c375e169f1afc2d9cce55928a15d594, regionState=CLOSING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:54,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, REOPEN/MOVE because future has completed 2024-11-24T04:21:54,008 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T04:21:54,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2c375e169f1afc2d9cce55928a15d594, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:21:54,176 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(122): Close 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:54,177 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T04:21:54,178 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1722): Closing 2c375e169f1afc2d9cce55928a15d594, disabling compactions & flushes 2024-11-24T04:21:54,178 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1755): Closing region test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:21:54,178 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:21:54,178 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1843): Acquired close lock on test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. after waiting 0 ms 2024-11-24T04:21:54,178 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1853): Updates disabled for region test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:21:54,182 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(2902): Flushing 2c375e169f1afc2d9cce55928a15d594 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-24T04:21:54,254 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/f/a73d9d62109f4ed1b1114d2cc76fdef3 is 37, key is row10/f:q1/1732422106681/Put/seqid=0 2024-11-24T04:21:54,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741848_1024 (size=8413) 2024-11-24T04:21:54,666 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/f/a73d9d62109f4ed1b1114d2cc76fdef3 2024-11-24T04:21:54,737 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/f/a73d9d62109f4ed1b1114d2cc76fdef3 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3 2024-11-24T04:21:54,749 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3, entries=99, sequenceid=103, filesize=8.2 K 2024-11-24T04:21:54,758 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 2c375e169f1afc2d9cce55928a15d594 in 573ms, sequenceid=103, compaction requested=false 2024-11-24T04:21:54,766 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2024-11-24T04:21:54,769 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:21:54,769 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1973): Closed test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:21:54,769 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1676): Region close journal for 2c375e169f1afc2d9cce55928a15d594: Waiting for close lock at 1732422114178Running coprocessor pre-close hooks at 1732422114178Disabling compacts and flushes for region at 1732422114178Disabling writes for close at 1732422114178Obtaining lock to block concurrent updates at 1732422114182 (+4 ms)Preparing flush snapshotting stores in 2c375e169f1afc2d9cce55928a15d594 at 1732422114183 (+1 ms)Finished memstore snapshotting test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594., syncing WAL and waiting on mvcc, flushsize=dataSize=3247, getHeapSize=11248, getOffHeapSize=0, getCellsCount=99 at 1732422114191 (+8 ms)Flushing stores of test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. at 1732422114193 (+2 ms)Flushing 2c375e169f1afc2d9cce55928a15d594/f: creating writer at 1732422114195 (+2 ms)Flushing 2c375e169f1afc2d9cce55928a15d594/f: appending metadata at 1732422114240 (+45 ms)Flushing 2c375e169f1afc2d9cce55928a15d594/f: closing flushed file at 1732422114243 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3df078a9: reopening flushed file at 1732422114735 (+492 ms)Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 2c375e169f1afc2d9cce55928a15d594 in 573ms, sequenceid=103, compaction requested=false at 1732422114758 (+23 ms)Writing region close event to WAL at 1732422114760 (+2 ms)Running coprocessor post-close hooks at 1732422114767 (+7 ms)Closed at 1732422114769 (+2 ms) 2024-11-24T04:21:54,769 WARN [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionServer(3299): Not adding moved region record: 2c375e169f1afc2d9cce55928a15d594 to self. 2024-11-24T04:21:54,773 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(157): Closed 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:54,774 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=2c375e169f1afc2d9cce55928a15d594, regionState=CLOSED 2024-11-24T04:21:54,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2c375e169f1afc2d9cce55928a15d594, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:21:54,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-24T04:21:54,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; CloseRegionProcedure 2c375e169f1afc2d9cce55928a15d594, server=e57b7116ff5e,36347,1732422095008 in 771 msec 2024-11-24T04:21:54,784 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, REOPEN/MOVE; state=CLOSED, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=true 2024-11-24T04:21:54,935 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=2c375e169f1afc2d9cce55928a15d594, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:54,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, REOPEN/MOVE because future has completed 2024-11-24T04:21:54,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c375e169f1afc2d9cce55928a15d594, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:21:55,096 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:21:55,097 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 2c375e169f1afc2d9cce55928a15d594, NAME => 'test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:21:55,097 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:21:55,097 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732422103966 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:55,097 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:55,097 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:55,098 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:55,099 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:55,100 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c375e169f1afc2d9cce55928a15d594 columnFamilyName f 2024-11-24T04:21:55,101 DEBUG [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:55,111 DEBUG [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3 2024-11-24T04:21:55,112 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.HStore(327): Store=2c375e169f1afc2d9cce55928a15d594/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:21:55,112 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:55,114 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c375e169f1afc2d9cce55928a15d594 columnFamilyName f3 2024-11-24T04:21:55,114 DEBUG [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:55,115 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.HStore(327): Store=2c375e169f1afc2d9cce55928a15d594/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:21:55,117 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:55,123 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c375e169f1afc2d9cce55928a15d594 columnFamilyName mob 2024-11-24T04:21:55,127 DEBUG [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:55,128 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.HStore(327): Store=2c375e169f1afc2d9cce55928a15d594/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:21:55,128 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:55,130 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:55,131 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:55,133 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:55,133 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:55,134 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1732422103966 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:21:55,136 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:55,138 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 2c375e169f1afc2d9cce55928a15d594; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9761276640, jitterRate=-0.09091027081012726}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:21:55,138 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:21:55,139 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 2c375e169f1afc2d9cce55928a15d594: Running coprocessor pre-open hook at 1732422115098Writing region info on filesystem at 1732422115098Initializing all the Stores at 1732422115099 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422115099Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422115099Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422115099Cleaning up temporary data from old regions at 1732422115133 (+34 ms)Running coprocessor post-open hooks at 1732422115138 (+5 ms)Region opened successfully at 1732422115139 (+1 ms) 2024-11-24T04:21:55,140 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594., pid=24, masterSystemTime=1732422115091 2024-11-24T04:21:55,143 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:21:55,143 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:21:55,144 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=2c375e169f1afc2d9cce55928a15d594, regionState=OPEN, openSeqNum=107, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:55,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c375e169f1afc2d9cce55928a15d594, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:21:55,151 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22 2024-11-24T04:21:55,151 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 2c375e169f1afc2d9cce55928a15d594, server=e57b7116ff5e,36347,1732422095008 in 210 msec 2024-11-24T04:21:55,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-24T04:21:55,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, REOPEN/MOVE in 1.1490 sec 2024-11-24T04:21:55,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-24T04:21:55,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1732422103966 in 1.1610 sec 2024-11-24T04:21:55,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1732422103966 in 1.5970 sec 2024-11-24T04:21:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=20 2024-11-24T04:21:55,168 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1732422103966 completed 2024-11-24T04:21:55,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f336ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:21:55,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:21:55,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:21:55,170 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:21:55,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:21:55,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:21:55,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@499bf0f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:21:55,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:21:55,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:21:55,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:21:55,172 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:21:55,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a68fd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:21:55,174 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:21:55,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:21:55,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:21:55,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:21:55,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732422103966', row='row-test-1732422103966-3-0', locateType=CURRENT is [region=test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594., hostname=e57b7116ff5e,36347,1732422095008, seqNum=107] 2024-11-24T04:21:55,230 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:21:55,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:21:55,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1db53a66 2024-11-24T04:21:55,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:21:55,235 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:21:55,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'backup'} 2024-11-24T04:21:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=backup 2024-11-24T04:21:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T04:21:55,263 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=backup in 11 msec 2024-11-24T04:21:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T04:21:55,361 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: backup completed 2024-11-24T04:21:55,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:21:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system 2024-11-24T04:21:55,389 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T04:21:55,389 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:55,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2024-11-24T04:21:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-24T04:21:55,390 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T04:21:55,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741849_1025 (size=635) 2024-11-24T04:21:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-24T04:21:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-24T04:21:55,802 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5a030641752b7def7c38c5f6295501b2, NAME => 'backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:21:55,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741850_1026 (size=40) 2024-11-24T04:21:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-24T04:21:56,214 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:56,215 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1722): Closing 5a030641752b7def7c38c5f6295501b2, disabling compactions & flushes 2024-11-24T04:21:56,215 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:21:56,215 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:21:56,215 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. after waiting 0 ms 2024-11-24T04:21:56,215 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:21:56,215 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1973): Closed backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:21:56,216 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5a030641752b7def7c38c5f6295501b2: Waiting for close lock at 1732422116214Disabling compacts and flushes for region at 1732422116214Disabling writes for close at 1732422116215 (+1 ms)Writing region close event to WAL at 1732422116215Closed at 1732422116215 2024-11-24T04:21:56,220 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T04:21:56,220 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1732422116220"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422116220"}]},"ts":"1732422116220"} 2024-11-24T04:21:56,225 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T04:21:56,227 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T04:21:56,228 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422116228"}]},"ts":"1732422116228"} 2024-11-24T04:21:56,231 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-24T04:21:56,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=5a030641752b7def7c38c5f6295501b2, ASSIGN}] 2024-11-24T04:21:56,233 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=5a030641752b7def7c38c5f6295501b2, ASSIGN 2024-11-24T04:21:56,234 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=5a030641752b7def7c38c5f6295501b2, ASSIGN; state=OFFLINE, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=false 2024-11-24T04:21:56,308 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17324221039663' 2024-11-24T04:21:56,309 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17324221039662' 2024-11-24T04:21:56,385 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=5a030641752b7def7c38c5f6295501b2, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:56,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=5a030641752b7def7c38c5f6295501b2, ASSIGN because future has completed 2024-11-24T04:21:56,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a030641752b7def7c38c5f6295501b2, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:21:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-24T04:21:56,546 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:21:56,546 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a030641752b7def7c38c5f6295501b2, NAME => 'backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:21:56,547 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:21:56,547 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:21:56,547 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:56,547 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:21:56,547 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:21:56,549 INFO [StoreOpener-5a030641752b7def7c38c5f6295501b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:21:56,551 INFO [StoreOpener-5a030641752b7def7c38c5f6295501b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a030641752b7def7c38c5f6295501b2 columnFamilyName meta 2024-11-24T04:21:56,551 DEBUG [StoreOpener-5a030641752b7def7c38c5f6295501b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:56,552 INFO [StoreOpener-5a030641752b7def7c38c5f6295501b2-1 {}] regionserver.HStore(327): Store=5a030641752b7def7c38c5f6295501b2/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:21:56,552 INFO [StoreOpener-5a030641752b7def7c38c5f6295501b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:21:56,554 INFO [StoreOpener-5a030641752b7def7c38c5f6295501b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a030641752b7def7c38c5f6295501b2 columnFamilyName session 2024-11-24T04:21:56,554 DEBUG [StoreOpener-5a030641752b7def7c38c5f6295501b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:56,555 INFO [StoreOpener-5a030641752b7def7c38c5f6295501b2-1 {}] regionserver.HStore(327): Store=5a030641752b7def7c38c5f6295501b2/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:21:56,555 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:21:56,556 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2 2024-11-24T04:21:56,557 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2 2024-11-24T04:21:56,558 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:21:56,558 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:21:56,558 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T04:21:56,560 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:21:56,563 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:21:56,564 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 5a030641752b7def7c38c5f6295501b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9830416160, jitterRate=-0.08447115123271942}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T04:21:56,564 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:21:56,564 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 5a030641752b7def7c38c5f6295501b2: Running coprocessor pre-open hook at 1732422116547Writing region info on filesystem at 1732422116547Initializing all the Stores at 1732422116548 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422116548Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422116549 (+1 ms)Cleaning up temporary data from old regions at 1732422116558 (+9 ms)Running coprocessor post-open hooks at 1732422116564 (+6 ms)Region opened successfully at 1732422116564 2024-11-24T04:21:56,566 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., pid=28, masterSystemTime=1732422116541 2024-11-24T04:21:56,568 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:21:56,568 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:21:56,569 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=5a030641752b7def7c38c5f6295501b2, regionState=OPEN, openSeqNum=2, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:56,571 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a030641752b7def7c38c5f6295501b2, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:21:56,575 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-24T04:21:56,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 5a030641752b7def7c38c5f6295501b2, server=e57b7116ff5e,36347,1732422095008 in 184 msec 2024-11-24T04:21:56,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-24T04:21:56,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=5a030641752b7def7c38c5f6295501b2, ASSIGN in 344 msec 2024-11-24T04:21:56,580 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T04:21:56,580 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422116580"}]},"ts":"1732422116580"} 2024-11-24T04:21:56,582 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-24T04:21:56,584 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T04:21:56,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system in 1.1990 sec 2024-11-24T04:21:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-24T04:21:57,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system completed 2024-11-24T04:21:57,558 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:21:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system_bulk 2024-11-24T04:21:57,561 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T04:21:57,561 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:57,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2024-11-24T04:21:57,563 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T04:21:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T04:21:57,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741851_1027 (size=640) 2024-11-24T04:21:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T04:21:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T04:21:57,973 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4317ba5909dca438c339af86075fb75a, NAME => 'backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:21:57,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741852_1028 (size=45) 2024-11-24T04:21:58,014 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-24T04:21:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T04:21:58,383 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:58,383 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1722): Closing 4317ba5909dca438c339af86075fb75a, disabling compactions & flushes 2024-11-24T04:21:58,383 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. 2024-11-24T04:21:58,383 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. 2024-11-24T04:21:58,384 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. after waiting 0 ms 2024-11-24T04:21:58,384 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. 2024-11-24T04:21:58,384 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1973): Closed backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. 2024-11-24T04:21:58,384 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4317ba5909dca438c339af86075fb75a: Waiting for close lock at 1732422118383Disabling compacts and flushes for region at 1732422118383Disabling writes for close at 1732422118384 (+1 ms)Writing region close event to WAL at 1732422118384Closed at 1732422118384 2024-11-24T04:21:58,451 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T04:21:58,451 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732422118451"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422118451"}]},"ts":"1732422118451"} 2024-11-24T04:21:58,454 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T04:21:58,456 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T04:21:58,456 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422118456"}]},"ts":"1732422118456"} 2024-11-24T04:21:58,459 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2024-11-24T04:21:58,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=4317ba5909dca438c339af86075fb75a, ASSIGN}] 2024-11-24T04:21:58,462 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=4317ba5909dca438c339af86075fb75a, ASSIGN 2024-11-24T04:21:58,463 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=4317ba5909dca438c339af86075fb75a, ASSIGN; state=OFFLINE, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=false 2024-11-24T04:21:58,614 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=4317ba5909dca438c339af86075fb75a, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:58,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=4317ba5909dca438c339af86075fb75a, ASSIGN because future has completed 2024-11-24T04:21:58,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4317ba5909dca438c339af86075fb75a, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:21:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T04:21:58,775 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. 2024-11-24T04:21:58,775 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 4317ba5909dca438c339af86075fb75a, NAME => 'backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:21:58,776 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:21:58,776 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 4317ba5909dca438c339af86075fb75a 2024-11-24T04:21:58,776 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:21:58,776 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 4317ba5909dca438c339af86075fb75a 2024-11-24T04:21:58,776 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 4317ba5909dca438c339af86075fb75a 2024-11-24T04:21:58,779 INFO [StoreOpener-4317ba5909dca438c339af86075fb75a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 4317ba5909dca438c339af86075fb75a 2024-11-24T04:21:58,782 INFO [StoreOpener-4317ba5909dca438c339af86075fb75a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4317ba5909dca438c339af86075fb75a columnFamilyName meta 2024-11-24T04:21:58,782 DEBUG [StoreOpener-4317ba5909dca438c339af86075fb75a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:58,783 INFO [StoreOpener-4317ba5909dca438c339af86075fb75a-1 {}] regionserver.HStore(327): Store=4317ba5909dca438c339af86075fb75a/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:21:58,783 INFO [StoreOpener-4317ba5909dca438c339af86075fb75a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 4317ba5909dca438c339af86075fb75a 2024-11-24T04:21:58,785 INFO [StoreOpener-4317ba5909dca438c339af86075fb75a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4317ba5909dca438c339af86075fb75a columnFamilyName session 2024-11-24T04:21:58,785 DEBUG [StoreOpener-4317ba5909dca438c339af86075fb75a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:21:58,786 INFO [StoreOpener-4317ba5909dca438c339af86075fb75a-1 {}] regionserver.HStore(327): Store=4317ba5909dca438c339af86075fb75a/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:21:58,786 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 4317ba5909dca438c339af86075fb75a 2024-11-24T04:21:58,787 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system_bulk/4317ba5909dca438c339af86075fb75a 2024-11-24T04:21:58,787 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system_bulk/4317ba5909dca438c339af86075fb75a 2024-11-24T04:21:58,788 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 4317ba5909dca438c339af86075fb75a 2024-11-24T04:21:58,788 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 4317ba5909dca438c339af86075fb75a 2024-11-24T04:21:58,789 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T04:21:58,790 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 4317ba5909dca438c339af86075fb75a 2024-11-24T04:21:58,793 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system_bulk/4317ba5909dca438c339af86075fb75a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:21:58,794 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 4317ba5909dca438c339af86075fb75a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12041860320, jitterRate=0.121485635638237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T04:21:58,794 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4317ba5909dca438c339af86075fb75a 2024-11-24T04:21:58,795 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 4317ba5909dca438c339af86075fb75a: Running coprocessor pre-open hook at 1732422118776Writing region info on filesystem at 1732422118776Initializing all the Stores at 1732422118778 (+2 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422118778Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422118779 (+1 ms)Cleaning up temporary data from old regions at 1732422118788 (+9 ms)Running coprocessor post-open hooks at 1732422118794 (+6 ms)Region opened successfully at 1732422118794 2024-11-24T04:21:58,796 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a., pid=31, masterSystemTime=1732422118769 2024-11-24T04:21:58,798 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. 2024-11-24T04:21:58,798 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. 2024-11-24T04:21:58,799 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=4317ba5909dca438c339af86075fb75a, regionState=OPEN, openSeqNum=2, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:21:58,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4317ba5909dca438c339af86075fb75a, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:21:58,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-24T04:21:58,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 4317ba5909dca438c339af86075fb75a, server=e57b7116ff5e,36347,1732422095008 in 186 msec 2024-11-24T04:21:58,811 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-24T04:21:58,811 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=4317ba5909dca438c339af86075fb75a, ASSIGN in 349 msec 2024-11-24T04:21:58,812 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T04:21:58,813 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422118812"}]},"ts":"1732422118812"} 2024-11-24T04:21:58,815 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2024-11-24T04:21:58,817 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T04:21:58,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system_bulk in 1.2590 sec 2024-11-24T04:21:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T04:21:59,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system_bulk completed 2024-11-24T04:21:59,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:21:59,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:21:59,734 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-24T04:21:59,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='activesession:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:21:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2024-11-24T04:21:59,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T04:21:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732422119758 (current time:1732422119758). 2024-11-24T04:21:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T04:21:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-24T04:21:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a3a29e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:21:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:21:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:21:59,760 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:21:59,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:21:59,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:21:59,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1506c4b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:21:59,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:21:59,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:21:59,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:21:59,762 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:21:59,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42ae320d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:21:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:21:59,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:21:59,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:21:59,768 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:21:59,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719. 2024-11-24T04:21:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:21:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:21:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:21:59,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:21:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T04:21:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T04:21:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-24T04:21:59,777 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T04:21:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T04:21:59,782 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T04:21:59,793 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T04:21:59,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741853_1029 (size=52) 2024-11-24T04:21:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T04:22:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T04:22:00,201 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T04:22:00,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2}] 2024-11-24T04:22:00,207 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:22:00,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36347 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-11-24T04:22:00,364 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:22:00,365 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 5a030641752b7def7c38c5f6295501b2 2/2 column families, dataSize=45 B heapSize=632 B 2024-11-24T04:22:00,383 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/session/e44333cb30954234bdc3a05f162b96cd is 49, key is activesession:/session:c/1732422119750/Put/seqid=0 2024-11-24T04:22:00,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741854_1030 (size=5049) 2024-11-24T04:22:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T04:22:00,789 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/session/e44333cb30954234bdc3a05f162b96cd 2024-11-24T04:22:00,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/session/e44333cb30954234bdc3a05f162b96cd as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/e44333cb30954234bdc3a05f162b96cd 2024-11-24T04:22:00,809 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/e44333cb30954234bdc3a05f162b96cd, entries=1, sequenceid=5, filesize=4.9 K 2024-11-24T04:22:00,811 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 5a030641752b7def7c38c5f6295501b2 in 446ms, sequenceid=5, compaction requested=false 2024-11-24T04:22:00,811 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-11-24T04:22:00,812 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 5a030641752b7def7c38c5f6295501b2: 2024-11-24T04:22:00,812 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. for snapshot_backup_system completed. 2024-11-24T04:22:00,813 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2.' region-info for snapshot=snapshot_backup_system 2024-11-24T04:22:00,815 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T04:22:00,817 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-24T04:22:00,818 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/e44333cb30954234bdc3a05f162b96cd] hfiles 2024-11-24T04:22:00,819 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/e44333cb30954234bdc3a05f162b96cd for snapshot=snapshot_backup_system 2024-11-24T04:22:00,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741855_1031 (size=95) 2024-11-24T04:22:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T04:22:01,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:22:01,231 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-24T04:22:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-24T04:22:01,234 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:22:01,234 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:22:01,238 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-24T04:22:01,238 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T04:22:01,238 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2 in 1.0330 sec 2024-11-24T04:22:01,240 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T04:22:01,241 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T04:22:01,241 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-24T04:22:01,243 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-24T04:22:01,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741856_1032 (size=731) 2024-11-24T04:22:01,664 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T04:22:01,678 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T04:22:01,679 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_backup_system 2024-11-24T04:22:01,681 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T04:22:01,681 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-24T04:22:01,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.9070 sec 2024-11-24T04:22:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T04:22:01,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-24T04:22:01,932 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732422115214 started at 1732422121931. 2024-11-24T04:22:01,949 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732422115214 has been started. 2024-11-24T04:22:01,954 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-24T04:22:01,961 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-11-24T04:22:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-24T04:22:01,963 INFO [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-24T04:22:01,964 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-24T04:22:01,964 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-24T04:22:01,964 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-24T04:22:01,965 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T04:22:01,965 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T04:22:02,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T04:22:02,039 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,040 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T04:22:02,040 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T04:22:02,041 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,041 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-24T04:22:02,042 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-24T04:22:02,043 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T04:22:02,045 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-24T04:22:02,045 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T04:22:02,046 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-24T04:22:02,049 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-24T04:22:02,049 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-24T04:22:02,050 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-24T04:22:02,050 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-24T04:22:02,050 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-24T04:22:02,050 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-24T04:22:02,050 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'e57b7116ff5e,36347,1732422095008' joining acquired barrier for procedure (rolllog) in zk 2024-11-24T04:22:02,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,060 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:02,060 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,060 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T04:22:02,061 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T04:22:02,061 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-24T04:22:02,061 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-24T04:22:02,061 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T04:22:02,062 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T04:22:02,063 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:02,063 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T04:22:02,064 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'e57b7116ff5e,36347,1732422095008' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-24T04:22:02,065 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@13caff8b[Count = 0] remaining members to acquire global barrier 2024-11-24T04:22:02,065 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-24T04:22:02,065 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:02,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:02,070 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:02,070 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:02,071 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,071 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-24T04:22:02,071 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-24T04:22:02,072 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-24T04:22:02,072 DEBUG [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,081 INFO [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732422098235 highest: 1732422105660 on e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,082 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0:(num 1732422098235) roll requested 2024-11-24T04:22:02,093 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422122082, exclude list is [], retry=0 2024-11-24T04:22:02,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:22:02,099 INFO [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422098235 with entries=3, filesize=1.00 KB; new WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422122082 2024-11-24T04:22:02,100 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:22:02,100 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422098235 is not closed yet, will try archiving it next time 2024-11-24T04:22:02,100 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL e57b7116ff5e%2C36347%2C1732422095008.meta:.meta(num 1732422098687) roll requested 2024-11-24T04:22:02,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741833_1009 (size=1034) 2024-11-24T04:22:02,104 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422098235 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422098235 2024-11-24T04:22:02,119 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422122100.meta, exclude list is [], retry=0 2024-11-24T04:22:02,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:22:02,125 INFO [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422098687.meta with entries=42, filesize=11.76 KB; new WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422122100.meta 2024-11-24T04:22:02,126 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:22:02,126 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422098687.meta is not closed yet, will try archiving it next time 2024-11-24T04:22:02,126 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1:(num 1732422105660) roll requested 2024-11-24T04:22:02,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741834_1010 (size=12055) 2024-11-24T04:22:02,128 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422098687.meta not finished, retry = 0 2024-11-24T04:22:02,139 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422122126, exclude list is [], retry=0 2024-11-24T04:22:02,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:22:02,142 DEBUG [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-24T04:22:02,142 INFO [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732422122082 on e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,143 DEBUG [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:22:02,143 DEBUG [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:22:02,143 DEBUG [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@42c33eba 2024-11-24T04:22:02,143 DEBUG [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:22:02,144 INFO [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422105660 with entries=24, filesize=4.64 KB; new WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422122126 2024-11-24T04:22:02,144 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60485, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T04:22:02,145 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:22:02,145 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422105660 is not closed yet, will try archiving it next time 2024-11-24T04:22:02,145 DEBUG [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:22:02,145 DEBUG [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:22:02,146 DEBUG [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:22:02,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741840_1016 (size=4756) 2024-11-24T04:22:02,147 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59779, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T04:22:02,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:22:02,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:22:02,169 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='rslogts:hdfs://localhost:45347/backupUT\x00', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:22:02,174 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-24T04:22:02,174 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'e57b7116ff5e,36347,1732422095008' in zk 2024-11-24T04:22:02,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,186 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-24T04:22:02,186 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T04:22:02,186 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,186 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-24T04:22:02,186 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T04:22:02,186 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T04:22:02,187 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T04:22:02,187 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T04:22:02,188 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:02,188 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,188 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T04:22:02,189 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:02,189 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,190 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'e57b7116ff5e,36347,1732422095008': 2024-11-24T04:22:02,190 INFO [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-24T04:22:02,190 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-24T04:22:02,190 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-24T04:22:02,190 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-24T04:22:02,191 INFO [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-24T04:22:02,191 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'e57b7116ff5e,36347,1732422095008' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-24T04:22:02,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T04:22:02,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T04:22:02,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T04:22:02,197 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-24T04:22:02,197 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-24T04:22:02,197 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T04:22:02,197 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T04:22:02,197 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T04:22:02,197 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T04:22:02,197 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T04:22:02,198 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T04:22:02,198 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,198 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:02,198 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T04:22:02,198 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T04:22:02,199 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:02,199 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,200 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,200 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T04:22:02,200 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:02,201 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T04:22:02,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T04:22:02,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T04:22:02,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:02,218 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T04:22:02,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T04:22:02,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:02,218 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T04:22:02,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:02,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T04:22:02,218 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T04:22:02,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-24T04:22:02,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-24T04:22:02,219 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T04:22:02,219 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T04:22:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-24T04:22:02,237 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1732422122236_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } 2024-11-24T04:22:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732422122237 (current time:1732422122237). 2024-11-24T04:22:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T04:22:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732422122236_default_test-1732422103966 VERSION not specified, setting to 2 2024-11-24T04:22:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1094ca14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:22:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:22:02,239 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:22:02,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:22:02,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:22:02,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a457d41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:02,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:22:02,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:22:02,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:02,241 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:22:02,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41bfe893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:22:02,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:22:02,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:22:02,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:22:02,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719. 2024-11-24T04:22:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:22:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:02,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:22:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T04:22:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732422122236_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } 2024-11-24T04:22:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732422122236_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-24T04:22:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T04:22:02,250 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732422122236_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T04:22:02,252 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732422122236_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T04:22:02,255 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732422122236_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T04:22:02,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741860_1036 (size=84) 2024-11-24T04:22:02,310 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2024-11-24T04:22:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T04:22:02,379 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T04:22:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T04:22:02,662 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732422122236_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T04:22:02,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 2c375e169f1afc2d9cce55928a15d594}] 2024-11-24T04:22:02,663 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:02,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-11-24T04:22:02,817 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:22:02,818 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 2c375e169f1afc2d9cce55928a15d594 3/3 column families, dataSize=654 B heapSize=2.25 KB 2024-11-24T04:22:02,841 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/f3/f9440cbd308f4fa3b08696b88b8a8b86 is 58, key is row-test-1732422103966-3-0/f3:q1/1732422115181/Put/seqid=0 2024-11-24T04:22:02,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741861_1037 (size=5387) 2024-11-24T04:22:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T04:22:03,251 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/f3/f9440cbd308f4fa3b08696b88b8a8b86 2024-11-24T04:22:03,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741862_1038 (size=4469) 2024-11-24T04:22:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T04:22:03,698 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/mob/682085848b1a4dc1a2b01cad48fbef3b is 59, key is row-test-1732422103966-3-0/mob:q1/1732422115197/Put/seqid=0 2024-11-24T04:22:03,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741863_1039 (size=5484) 2024-11-24T04:22:04,155 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=330, hasBloomFilter=true, into tmp file hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/mob/682085848b1a4dc1a2b01cad48fbef3b 2024-11-24T04:22:04,165 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/f3/f9440cbd308f4fa3b08696b88b8a8b86 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f3/f9440cbd308f4fa3b08696b88b8a8b86 2024-11-24T04:22:04,174 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f3/f9440cbd308f4fa3b08696b88b8a8b86, entries=6, sequenceid=121, filesize=5.3 K 2024-11-24T04:22:04,176 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/mob/682085848b1a4dc1a2b01cad48fbef3b as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/682085848b1a4dc1a2b01cad48fbef3b 2024-11-24T04:22:04,186 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/682085848b1a4dc1a2b01cad48fbef3b, entries=6, sequenceid=121, filesize=5.4 K 2024-11-24T04:22:04,187 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~654 B/654, heapSize ~1.97 KB/2016, currentSize=0 B/0 for 2c375e169f1afc2d9cce55928a15d594 in 1370ms, sequenceid=121, compaction requested=false 2024-11-24T04:22:04,187 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 2c375e169f1afc2d9cce55928a15d594: 2024-11-24T04:22:04,187 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. for snapshot_1732422122236_default_test-1732422103966 completed. 2024-11-24T04:22:04,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594.' region-info for snapshot=snapshot_1732422122236_default_test-1732422103966 2024-11-24T04:22:04,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T04:22:04,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3] hfiles 2024-11-24T04:22:04,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3 for snapshot=snapshot_1732422122236_default_test-1732422103966 2024-11-24T04:22:04,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f3/f9440cbd308f4fa3b08696b88b8a8b86] hfiles 2024-11-24T04:22:04,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f3/f9440cbd308f4fa3b08696b88b8a8b86 for snapshot=snapshot_1732422122236_default_test-1732422103966 2024-11-24T04:22:04,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/682085848b1a4dc1a2b01cad48fbef3b] hfiles 2024-11-24T04:22:04,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/682085848b1a4dc1a2b01cad48fbef3b for snapshot=snapshot_1732422122236_default_test-1732422103966 2024-11-24T04:22:04,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741864_1040 (size=185) 2024-11-24T04:22:04,279 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-24T04:22:04,321 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-24T04:22:04,321 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2024-11-24T04:22:04,322 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-11-24T04:22:04,322 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2024-11-24T04:22:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T04:22:04,596 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:22:04,596 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-24T04:22:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-24T04:22:04,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732422122236_default_test-1732422103966 on region 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:04,597 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:04,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-24T04:22:04,601 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732422122236_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T04:22:04,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 2c375e169f1afc2d9cce55928a15d594 in 1.9360 sec 2024-11-24T04:22:04,602 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732422122236_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T04:22:04,604 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-24T04:22:04,604 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-24T04:22:04,605 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:22:04,605 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-24T04:22:04,605 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:22:04,606 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f3 2024-11-24T04:22:04,606 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:22:04,606 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: mob 2024-11-24T04:22:04,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741865_1041 (size=49) 2024-11-24T04:22:05,018 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732422122236_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T04:22:05,018 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732422122236_default_test-1732422103966 2024-11-24T04:22:05,021 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_1732422122236_default_test-1732422103966 2024-11-24T04:22:05,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741866_1042 (size=1146) 2024-11-24T04:22:05,436 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732422122236_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T04:22:05,445 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732422122236_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T04:22:05,446 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_1732422122236_default_test-1732422103966 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_1732422122236_default_test-1732422103966 2024-11-24T04:22:05,448 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732422122236_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T04:22:05,448 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732422122236_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-24T04:22:05,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732422122236_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } in 3.2010 sec 2024-11-24T04:22:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T04:22:06,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1732422103966 completed 2024-11-24T04:22:06,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1732422126421_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } 2024-11-24T04:22:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732422126422 (current time:1732422126422). 2024-11-24T04:22:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T04:22:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732422126421_ns2_test-17324221039661 VERSION not specified, setting to 2 2024-11-24T04:22:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d6302e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:22:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:22:06,425 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:22:06,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:22:06,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:22:06,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47229155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:06,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:22:06,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:22:06,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:06,427 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:22:06,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c3aa954, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:22:06,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:22:06,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:22:06,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:22:06,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719. 2024-11-24T04:22:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:22:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:06,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:22:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T04:22:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732422126421_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } 2024-11-24T04:22:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732422126421_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-24T04:22:06,439 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732422126421_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T04:22:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-24T04:22:06,440 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732422126421_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T04:22:06,444 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732422126421_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T04:22:06,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741867_1043 (size=86) 2024-11-24T04:22:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-24T04:22:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-24T04:22:06,851 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732422126421_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T04:22:06,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d3f34500755154053262a8d9790ac0a1}] 2024-11-24T04:22:06,852 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d3f34500755154053262a8d9790ac0a1 2024-11-24T04:22:07,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2024-11-24T04:22:07,005 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:22:07,005 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing d3f34500755154053262a8d9790ac0a1 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-24T04:22:07,029 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/.tmp/f/933ab35b68994dc8be3ccc706f0c0aa9 is 37, key is row10/f:q1/1732422109020/Put/seqid=0 2024-11-24T04:22:07,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741868_1044 (size=8413) 2024-11-24T04:22:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-24T04:22:07,436 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/.tmp/f/933ab35b68994dc8be3ccc706f0c0aa9 2024-11-24T04:22:07,451 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/.tmp/f/933ab35b68994dc8be3ccc706f0c0aa9 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/933ab35b68994dc8be3ccc706f0c0aa9 2024-11-24T04:22:07,461 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/933ab35b68994dc8be3ccc706f0c0aa9, entries=99, sequenceid=103, filesize=8.2 K 2024-11-24T04:22:07,462 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for d3f34500755154053262a8d9790ac0a1 in 456ms, sequenceid=103, compaction requested=false 2024-11-24T04:22:07,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for d3f34500755154053262a8d9790ac0a1: 2024-11-24T04:22:07,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. for snapshot_1732422126421_ns2_test-17324221039661 completed. 2024-11-24T04:22:07,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1.' region-info for snapshot=snapshot_1732422126421_ns2_test-17324221039661 2024-11-24T04:22:07,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T04:22:07,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/933ab35b68994dc8be3ccc706f0c0aa9] hfiles 2024-11-24T04:22:07,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/933ab35b68994dc8be3ccc706f0c0aa9 for snapshot=snapshot_1732422126421_ns2_test-17324221039661 2024-11-24T04:22:07,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741869_1045 (size=91) 2024-11-24T04:22:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-24T04:22:07,874 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:22:07,874 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-24T04:22:07,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-24T04:22:07,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732422126421_ns2_test-17324221039661 on region d3f34500755154053262a8d9790ac0a1 2024-11-24T04:22:07,876 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d3f34500755154053262a8d9790ac0a1 2024-11-24T04:22:07,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-24T04:22:07,881 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732422126421_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T04:22:07,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; SnapshotRegionProcedure d3f34500755154053262a8d9790ac0a1 in 1.0260 sec 2024-11-24T04:22:07,882 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732422126421_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T04:22:07,883 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732422126421_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T04:22:07,883 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732422126421_ns2_test-17324221039661 2024-11-24T04:22:07,885 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_1732422126421_ns2_test-17324221039661 2024-11-24T04:22:07,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741870_1046 (size=456) 2024-11-24T04:22:08,297 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732422126421_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T04:22:08,304 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732422126421_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T04:22:08,304 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_1732422126421_ns2_test-17324221039661 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_1732422126421_ns2_test-17324221039661 2024-11-24T04:22:08,306 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732422126421_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T04:22:08,306 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732422126421_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-24T04:22:08,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732422126421_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } in 1.8690 sec 2024-11-24T04:22:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-24T04:22:08,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17324221039661 completed 2024-11-24T04:22:08,601 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1732422115214 2024-11-24T04:22:08,601 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-24T04:22:08,602 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2024-11-24T04:22:08,603 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732422115214_ns2:test-17324221039661 2024-11-24T04:22:08,603 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732422126421_ns2_test-17324221039661 to hdfs://localhost:45347/backupUT/backup_1732422115214/ns2/test-17324221039661/ 2024-11-24T04:22:08,607 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-24T04:22:08,638 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:45347, inputRoot=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:22:08,638 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1702068436_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45347/backupUT/backup_1732422115214/ns2/test-17324221039661, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45347/backupUT/backup_1732422115214/ns2/test-17324221039661/.hbase-snapshot/.tmp/snapshot_1732422126421_ns2_test-17324221039661 2024-11-24T04:22:08,641 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-24T04:22:08,646 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_1732422126421_ns2_test-17324221039661 to hdfs://localhost:45347/backupUT/backup_1732422115214/ns2/test-17324221039661/.hbase-snapshot/.tmp/snapshot_1732422126421_ns2_test-17324221039661 2024-11-24T04:22:08,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741871_1047 (size=456) 2024-11-24T04:22:08,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741872_1048 (size=86) 2024-11-24T04:22:09,067 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:09,068 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:09,068 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:09,973 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-17949264845504074513.jar 2024-11-24T04:22:09,973 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:09,973 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:09,974 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:09,974 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:09,974 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:09,974 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:09,975 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:09,975 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:09,975 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:09,975 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:22:09,976 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T04:22:09,976 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T04:22:09,976 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T04:22:09,976 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T04:22:09,977 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T04:22:09,977 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T04:22:09,977 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T04:22:09,977 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T04:22:09,978 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T04:22:09,978 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T04:22:09,980 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:09,980 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:09,980 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:09,980 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:09,980 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:09,981 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:09,981 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:10,039 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:22:10,065 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732422126421_ns2_test-17324221039661' hfile list 2024-11-24T04:22:10,070 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=8.2 K 2024-11-24T04:22:10,269 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local1731781727_0001_ac91c753-883f-4553-85f0-f38546449e52/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-24T04:22:10,270 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local1731781727_0001_ac91c753-883f-4553-85f0-f38546449e52/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-24T04:22:10,413 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-24T04:22:10,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741873_1049 (size=8413) 2024-11-24T04:22:10,895 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/933ab35b68994dc8be3ccc706f0c0aa9 output=hdfs://localhost:45347/backupUT/backup_1732422115214/ns2/test-17324221039661/archive/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/933ab35b68994dc8be3ccc706f0c0aa9 2024-11-24T04:22:10,897 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.017M/sec 2024-11-24T04:22:11,349 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-24T04:22:11,351 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-24T04:22:11,357 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732422126421_ns2_test-17324221039661 2024-11-24T04:22:11,357 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732422126421_ns2_test-17324221039661 finished. 2024-11-24T04:22:11,358 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732422115214_test-1732422103966 2024-11-24T04:22:11,358 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732422122236_default_test-1732422103966 to hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/ 2024-11-24T04:22:11,358 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-24T04:22:11,385 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:45347, inputRoot=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:22:11,385 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1702068436_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/.hbase-snapshot/.tmp/snapshot_1732422122236_default_test-1732422103966 2024-11-24T04:22:11,387 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-24T04:22:11,394 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_1732422122236_default_test-1732422103966 to hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/.hbase-snapshot/.tmp/snapshot_1732422122236_default_test-1732422103966 2024-11-24T04:22:11,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741875_1051 (size=1146) 2024-11-24T04:22:11,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741874_1050 (size=84) 2024-11-24T04:22:11,811 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:11,812 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:11,812 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:12,697 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-15703836476368487679.jar 2024-11-24T04:22:12,697 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:12,697 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:12,698 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:12,698 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:12,698 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:12,698 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:12,698 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:12,699 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:12,699 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:12,699 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:22:12,699 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T04:22:12,699 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T04:22:12,700 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T04:22:12,700 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T04:22:12,700 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T04:22:12,700 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T04:22:12,701 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T04:22:12,701 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T04:22:12,701 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T04:22:12,701 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T04:22:12,702 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:12,702 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:12,702 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:12,702 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:12,702 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:12,702 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:12,703 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:12,739 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:22:12,763 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732422122236_default_test-1732422103966' hfile list 2024-11-24T04:22:12,766 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=18.8 K 2024-11-24T04:22:12,848 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local394838184_0002_74bba30c-6626-4024-a0e5-2ecfc5182ab1/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-24T04:22:12,848 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local394838184_0002_74bba30c-6626-4024-a0e5-2ecfc5182ab1/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-24T04:22:12,934 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-24T04:22:12,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741876_1052 (size=8413) 2024-11-24T04:22:13,370 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3 output=hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3 2024-11-24T04:22:13,370 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.019M/sec 2024-11-24T04:22:13,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741877_1053 (size=5484) 2024-11-24T04:22:13,829 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/682085848b1a4dc1a2b01cad48fbef3b output=hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/682085848b1a4dc1a2b01cad48fbef3b 2024-11-24T04:22:13,829 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-11-24T04:22:13,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741878_1054 (size=5387) 2024-11-24T04:22:14,282 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f3/f9440cbd308f4fa3b08696b88b8a8b86 output=hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f3/f9440cbd308f4fa3b08696b88b8a8b86 2024-11-24T04:22:14,282 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5387 (5.3 K) time=0sec 0.011M/sec 2024-11-24T04:22:14,910 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-24T04:22:14,911 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-24T04:22:14,920 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732422122236_default_test-1732422103966 2024-11-24T04:22:14,920 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732422122236_default_test-1732422103966 finished. 2024-11-24T04:22:14,921 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17324221039661 2024-11-24T04:22:14,921 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1732422103966 2024-11-24T04:22:14,965 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732422115214 2024-11-24T04:22:14,965 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-24T04:22:14,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741879_1055 (size=130) 2024-11-24T04:22:15,376 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45347/backupUT/backup_1732422115214/.backup.manifest 2024-11-24T04:22:15,380 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732422115214 finished: type=FULL,tablelist=ns2:test-17324221039661;test-1732422103966,targetRootDir=hdfs://localhost:45347/backupUT,startts=1732422121931,completets=1732422134963,bytescopied=0 2024-11-24T04:22:15,380 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-24T04:22:15,380 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732422126421_ns2_test-17324221039661 2024-11-24T04:22:15,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1732422126421_ns2_test-17324221039661" type: DISABLED 2024-11-24T04:22:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732422126421_ns2_test-17324221039661 2024-11-24T04:22:15,392 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732422126421_ns2_test-17324221039661 for backup backup_1732422115214 succeeded. 2024-11-24T04:22:15,392 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732422122236_default_test-1732422103966 2024-11-24T04:22:15,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1732422122236_default_test-1732422103966" type: DISABLED 2024-11-24T04:22:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732422122236_default_test-1732422103966 2024-11-24T04:22:15,397 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732422122236_default_test-1732422103966 for backup backup_1732422115214 succeeded. 2024-11-24T04:22:15,398 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-24T04:22:15,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-11-24T04:22:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-24T04:22:15,413 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-24T04:22:15,418 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-24T04:22:15,431 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732422115214 completed. 2024-11-24T04:22:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:22:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:22:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:backup_1732422115214', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:22:15,648 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(113): writing 99 rows to test-1732422103966 2024-11-24T04:22:15,669 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(116): written 99 rows to test-1732422103966 2024-11-24T04:22:15,860 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(120): written 111 rows to test-1732422103966 to Mob enabled CF 2024-11-24T04:22:15,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17324221039661', row='row-t20', locateType=CURRENT is [region=ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:22:15,898 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(133): written 5 rows to ns2:test-17324221039661 2024-11-24T04:22:15,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$3(2313): Client=jenkins//172.17.0.3 split test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:22:15,907 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] assignment.SplitTableRegionProcedure(210): test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. splitKey isn't explicitly specified, will try to find a best split key from RS e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:15,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3 because midkey is the same as first or last row 2024-11-24T04:22:15,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f3/f9440cbd308f4fa3b08696b88b8a8b86 because midkey is the same as first or last row 2024-11-24T04:22:15,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/682085848b1a4dc1a2b01cad48fbef3b because midkey is the same as first or last row 2024-11-24T04:22:15,911 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.HRegion(2902): Flushing 2c375e169f1afc2d9cce55928a15d594 3/3 column families, dataSize=11.48 KB heapSize=27.79 KB 2024-11-24T04:22:15,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/f/60684cdd8d5e45cbaabfe6505beaad9f is 59, key is row-test-1732422103966-1-10/f:q1/1732422135476/Put/seqid=0 2024-11-24T04:22:15,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741880_1056 (size=10796) 2024-11-24T04:22:16,337 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/f/60684cdd8d5e45cbaabfe6505beaad9f 2024-11-24T04:22:16,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124065620beac114f4a94d99cd6792a819b_2c375e169f1afc2d9cce55928a15d594 is 63, key is row-test-1732422103966-3-100/mob:q1/1732422135842/Put/seqid=0 2024-11-24T04:22:16,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741881_1057 (size=5591) 2024-11-24T04:22:16,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:22:16,783 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124065620beac114f4a94d99cd6792a819b_2c375e169f1afc2d9cce55928a15d594 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/mobdir/data/default/test-1732422103966/3047fc58bb55744b4303d1cceb01bae9/mob/d41d8cd98f00b204e9800998ecf8427e20241124065620beac114f4a94d99cd6792a819b_2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:16,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/mob/4adb0dbc87f947bdaad87e7746c93da8, store: [table=test-1732422103966 family=mob region=2c375e169f1afc2d9cce55928a15d594] 2024-11-24T04:22:16,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/mob/4adb0dbc87f947bdaad87e7746c93da8 is 192, key is row-test-1732422103966-3-100/mob:q1/1732422135842/Put/seqid=0 2024-11-24T04:22:16,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741882_1058 (size=13760) 2024-11-24T04:22:17,191 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=6.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/mob/4adb0dbc87f947bdaad87e7746c93da8 2024-11-24T04:22:17,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/f/60684cdd8d5e45cbaabfe6505beaad9f as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/60684cdd8d5e45cbaabfe6505beaad9f 2024-11-24T04:22:17,218 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/60684cdd8d5e45cbaabfe6505beaad9f, entries=99, sequenceid=334, filesize=10.5 K 2024-11-24T04:22:17,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/mob/4adb0dbc87f947bdaad87e7746c93da8 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/4adb0dbc87f947bdaad87e7746c93da8 2024-11-24T04:22:17,226 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/4adb0dbc87f947bdaad87e7746c93da8, entries=111, sequenceid=334, filesize=13.4 K 2024-11-24T04:22:17,227 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.48 KB/11754, heapSize ~27.51 KB/28168, currentSize=0 B/0 for 2c375e169f1afc2d9cce55928a15d594 in 1316ms, sequenceid=334, compaction requested=false 2024-11-24T04:22:17,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.HRegion(2603): Flush status journal for 2c375e169f1afc2d9cce55928a15d594: 2024-11-24T04:22:17,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/60684cdd8d5e45cbaabfe6505beaad9f because midkey is the same as first or last row 2024-11-24T04:22:17,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f3/f9440cbd308f4fa3b08696b88b8a8b86 because midkey is the same as first or last row 2024-11-24T04:22:17,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/4adb0dbc87f947bdaad87e7746c93da8 because midkey is the same as first or last row 2024-11-24T04:22:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T04:22:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.CallRunner(138): callId: 303 service: MasterService methodName: SplitRegion size: 99 connection: 172.17.0.3:48510 deadline: 1732422195905, exception=org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row 2024-11-24T04:22:17,235 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(144): region is not splittable, because java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:22:17,235 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: SPLIT_REGION, Table Name: default:test-1732422103966 failed with org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:22:17,235 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T04:22:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732422103966,, stopping at row=test-1732422103966 ,, for max=2147483647 with caching=100 2024-11-24T04:22:17,245 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732422103966,, stopping at row=test-1732422103966 ,, for max=2147483647 with caching=100 2024-11-24T04:22:17,253 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(149): split finished in =1355 2024-11-24T04:22:17,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:22:17,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:22:17,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:22:17,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:22:17,291 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-24T04:22:17,295 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732422137253 2024-11-24T04:22:17,301 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732422115214} 2024-11-24T04:22:17,301 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-24T04:22:17,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:22:17,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:22:17,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:22:17,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:22:17,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T04:22:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732422137358 (current time:1732422137358). 2024-11-24T04:22:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T04:22:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-24T04:22:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af1711c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:22:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:22:17,360 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:22:17,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:22:17,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:22:17,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ce64ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:17,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:22:17,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:22:17,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:17,363 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:22:17,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@301a63f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:22:17,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:22:17,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:22:17,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:22:17,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719. 2024-11-24T04:22:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:22:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:17,367 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:22:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T04:22:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T04:22:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-24T04:22:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-24T04:22:17,370 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T04:22:17,371 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T04:22:17,373 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T04:22:17,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741883_1059 (size=52) 2024-11-24T04:22:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-24T04:22:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-24T04:22:17,780 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T04:22:17,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2}] 2024-11-24T04:22:17,782 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:22:17,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36347 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2024-11-24T04:22:17,936 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:22:17,937 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2902): Flushing 5a030641752b7def7c38c5f6295501b2 2/2 column families, dataSize=1.44 KB heapSize=2.75 KB 2024-11-24T04:22:17,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/meta/d61b2831ca274cc7989a43ee1404fb09 is 158, key is trslm:hdfs://localhost:45347/backupUT\x00ns2:test-17324221039661/meta:log-roll-map/1732422134952/Put/seqid=0 2024-11-24T04:22:17,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741884_1060 (size=5914) 2024-11-24T04:22:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-24T04:22:18,367 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/meta/d61b2831ca274cc7989a43ee1404fb09 2024-11-24T04:22:18,381 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/session/fea54c00fd9144c2b57bfc7379644a05 is 433, key is session:backup_1732422115214/session:context/1732422135417/Put/seqid=0 2024-11-24T04:22:18,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741885_1061 (size=5535) 2024-11-24T04:22:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-24T04:22:18,786 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/session/fea54c00fd9144c2b57bfc7379644a05 2024-11-24T04:22:18,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/meta/d61b2831ca274cc7989a43ee1404fb09 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/d61b2831ca274cc7989a43ee1404fb09 2024-11-24T04:22:18,808 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/d61b2831ca274cc7989a43ee1404fb09, entries=6, sequenceid=17, filesize=5.8 K 2024-11-24T04:22:18,809 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/session/fea54c00fd9144c2b57bfc7379644a05 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/fea54c00fd9144c2b57bfc7379644a05 2024-11-24T04:22:18,816 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/fea54c00fd9144c2b57bfc7379644a05, entries=2, sequenceid=17, filesize=5.4 K 2024-11-24T04:22:18,817 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3140): Finished flush of dataSize ~1.44 KB/1478, heapSize ~2.72 KB/2784, currentSize=0 B/0 for 5a030641752b7def7c38c5f6295501b2 in 881ms, sequenceid=17, compaction requested=false 2024-11-24T04:22:18,817 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2603): Flush status journal for 5a030641752b7def7c38c5f6295501b2: 2024-11-24T04:22:18,817 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. for snapshot_backup_system completed. 2024-11-24T04:22:18,817 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2.' region-info for snapshot=snapshot_backup_system 2024-11-24T04:22:18,817 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T04:22:18,817 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/d61b2831ca274cc7989a43ee1404fb09] hfiles 2024-11-24T04:22:18,817 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/d61b2831ca274cc7989a43ee1404fb09 for snapshot=snapshot_backup_system 2024-11-24T04:22:18,817 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/e44333cb30954234bdc3a05f162b96cd, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/fea54c00fd9144c2b57bfc7379644a05] hfiles 2024-11-24T04:22:18,817 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/e44333cb30954234bdc3a05f162b96cd for snapshot=snapshot_backup_system 2024-11-24T04:22:18,817 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/fea54c00fd9144c2b57bfc7379644a05 for snapshot=snapshot_backup_system 2024-11-24T04:22:18,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741886_1062 (size=173) 2024-11-24T04:22:18,961 DEBUG [master/e57b7116ff5e:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732422103966 2024-11-24T04:22:18,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741887_1063 (size=950) 2024-11-24T04:22:19,225 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:22:19,225 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-11-24T04:22:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-11-24T04:22:19,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:22:19,228 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:22:19,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-24T04:22:19,235 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T04:22:19,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2 in 1.4490 sec 2024-11-24T04:22:19,237 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T04:22:19,239 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T04:22:19,239 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-24T04:22:19,241 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-24T04:22:19,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741888_1064 (size=810) 2024-11-24T04:22:19,374 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/.tabledesc/.tableinfo.0000000002.908 2024-11-24T04:22:19,374 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/.tabledesc/.tableinfo.0000000003.950 2024-11-24T04:22:19,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732422103966 in 418 msec 2024-11-24T04:22:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-24T04:22:19,658 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T04:22:19,670 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T04:22:19,670 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_backup_system 2024-11-24T04:22:19,672 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T04:22:19,672 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-24T04:22:19,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.3040 sec 2024-11-24T04:22:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-24T04:22:21,533 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-24T04:22:21,533 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732422137253 started at 1732422141533. 2024-11-24T04:22:21,540 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732422137253 has been started. 2024-11-24T04:22:21,549 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [ns2:test-17324221039661, test-1732422103966] 2024-11-24T04:22:21,554 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1732422105660for backupID backup_1732422137253 2024-11-24T04:22:21,554 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-24T04:22:21,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-11-24T04:22:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-24T04:22:21,555 INFO [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-24T04:22:21,555 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-24T04:22:21,555 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-24T04:22:21,556 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-24T04:22:21,556 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T04:22:21,557 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T04:22:21,574 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T04:22:21,574 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T04:22:21,574 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T04:22:21,575 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,575 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-24T04:22:21,575 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-24T04:22:21,575 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T04:22:21,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-24T04:22:21,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T04:22:21,576 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-24T04:22:21,576 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-24T04:22:21,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-24T04:22:21,576 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-24T04:22:21,576 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-24T04:22:21,577 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-24T04:22:21,577 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-24T04:22:21,577 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'e57b7116ff5e,36347,1732422095008' joining acquired barrier for procedure (rolllog) in zk 2024-11-24T04:22:21,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,584 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:21,585 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,585 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T04:22:21,585 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T04:22:21,585 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-24T04:22:21,585 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-24T04:22:21,585 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T04:22:21,586 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T04:22:21,586 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:21,587 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,587 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T04:22:21,588 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'e57b7116ff5e,36347,1732422095008' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-24T04:22:21,588 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@278efd63[Count = 0] remaining members to acquire global barrier 2024-11-24T04:22:21,588 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-24T04:22:21,588 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:21,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:21,595 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:21,595 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:21,595 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-24T04:22:21,595 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,596 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-24T04:22:21,596 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-24T04:22:21,596 DEBUG [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,596 INFO [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732422122082 highest: 1732422122126 on e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,596 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0:(num 1732422122082) roll requested 2024-11-24T04:22:21,608 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422141596, exclude list is [], retry=0 2024-11-24T04:22:21,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:22:21,613 INFO [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422122082 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422141596 2024-11-24T04:22:21,613 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:22:21,613 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422122082 is not closed yet, will try archiving it next time 2024-11-24T04:22:21,614 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL e57b7116ff5e%2C36347%2C1732422095008.meta:.meta(num 1732422122100) roll requested 2024-11-24T04:22:21,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741857_1033 (size=1198) 2024-11-24T04:22:21,615 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422122082 not finished, retry = 0 2024-11-24T04:22:21,626 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422141614.meta, exclude list is [], retry=0 2024-11-24T04:22:21,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:22:21,630 INFO [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422122100.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422141614.meta 2024-11-24T04:22:21,630 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:22:21,631 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422122100.meta is not closed yet, will try archiving it next time 2024-11-24T04:22:21,631 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1:(num 1732422122126) roll requested 2024-11-24T04:22:21,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741858_1034 (size=93) 2024-11-24T04:22:21,632 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422122100.meta not finished, retry = 0 2024-11-24T04:22:21,644 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422141631, exclude list is [], retry=0 2024-11-24T04:22:21,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:22:21,648 INFO [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422122126 with entries=224, filesize=29.77 KB; new WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422141631 2024-11-24T04:22:21,649 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:22:21,649 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422122126 is not closed yet, will try archiving it next time 2024-11-24T04:22:21,649 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422105660 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422105660 2024-11-24T04:22:21,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741859_1035 (size=30490) 2024-11-24T04:22:21,650 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422122126 not finished, retry = 0 2024-11-24T04:22:21,656 DEBUG [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-24T04:22:21,656 INFO [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732422141596 on e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:22:21,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:22:21,678 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-24T04:22:21,678 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'e57b7116ff5e,36347,1732422095008' in zk 2024-11-24T04:22:21,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,690 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-24T04:22:21,690 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T04:22:21,690 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-24T04:22:21,690 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,690 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T04:22:21,690 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T04:22:21,691 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T04:22:21,691 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T04:22:21,692 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:21,692 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,693 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T04:22:21,693 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:21,694 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,695 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'e57b7116ff5e,36347,1732422095008': 2024-11-24T04:22:21,695 INFO [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-24T04:22:21,695 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-24T04:22:21,695 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'e57b7116ff5e,36347,1732422095008' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-24T04:22:21,695 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-24T04:22:21,695 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-24T04:22:21,695 INFO [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-24T04:22:21,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T04:22:21,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T04:22:21,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T04:22:21,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-24T04:22:21,701 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-24T04:22:21,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T04:22:21,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T04:22:21,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T04:22:21,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T04:22:21,701 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T04:22:21,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T04:22:21,701 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,702 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:21,702 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T04:22:21,702 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T04:22:21,702 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:21,703 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,703 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,703 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T04:22:21,703 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:21,704 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T04:22:21,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T04:22:21,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T04:22:21,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:21,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T04:22:21,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:21,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:21,711 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T04:22:21,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T04:22:21,712 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T04:22:21,712 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T04:22:21,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-24T04:22:21,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-24T04:22:21,712 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T04:22:21,712 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T04:22:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-24T04:22:21,724 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {e57b7116ff5e:36347=1732422105660} newestTimestamps: {e57b7116ff5e:36347=1732422122126} 2024-11-24T04:22:21,727 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: e57b7116ff5e%2C36347%2C1732422095008.meta.1732422098687.meta 2024-11-24T04:22:21,727 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: e57b7116ff5e%2C36347%2C1732422095008.meta.1732422122100.meta 2024-11-24T04:22:21,727 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: e57b7116ff5e%2C36347%2C1732422095008.meta.1732422141614.meta 2024-11-24T04:22:21,727 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422122082 2024-11-24T04:22:21,727 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422141596 2024-11-24T04:22:21,727 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422122126 2024-11-24T04:22:21,727 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422141631 2024-11-24T04:22:21,733 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422122100.meta to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422122100.meta 2024-11-24T04:22:21,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741892_1068 (size=364) 2024-11-24T04:22:22,141 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17324221039661 target: hdfs://localhost:45347/backupUT/backup_1732422137253/ns2/test-17324221039661 descriptor: 'ns2:test-17324221039661', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:22:22,141 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-24T04:22:22,145 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17324221039661 2024-11-24T04:22:22,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741893_1069 (size=50) 2024-11-24T04:22:22,555 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17324221039661 2024-11-24T04:22:22,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741894_1070 (size=950) 2024-11-24T04:22:22,977 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1732422103966 target: hdfs://localhost:45347/backupUT/backup_1732422137253/default/test-1732422103966 descriptor: 'test-1732422103966', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:22:22,977 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-24T04:22:22,985 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1732422103966 2024-11-24T04:22:22,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741895_1071 (size=53) 2024-11-24T04:22:23,392 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1732422103966 2024-11-24T04:22:23,405 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:45347/backupUT/.tmp/backup_1732422137253 from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422122082;hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422122126 2024-11-24T04:22:23,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@715d4dcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:23,407 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:22:23,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:22:23,408 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:22:23,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:22:23,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:22:23,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b5fd0f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:23,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:22:23,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:22:23,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:23,410 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:22:23,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f8b11e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:23,411 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:22:23,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:22:23,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4b04b952 2024-11-24T04:22:23,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:22:23,412 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:22:23,418 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-24T04:22:23,418 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17324221039661 2024-11-24T04:22:23,418 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17324221039661,, stopping at row=ns2:test-17324221039661 ,, for max=2147483647 with caching=100 2024-11-24T04:22:23,419 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:22:23,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:22:23,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:22:23,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:22:23,423 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17324221039661: ns2:test-17324221039661; 2024-11-24T04:22:23,424 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1732422103966 2024-11-24T04:22:23,424 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732422103966,, stopping at row=test-1732422103966 ,, for max=2147483647 with caching=100 2024-11-24T04:22:23,426 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1732422103966: test-1732422103966; 2024-11-24T04:22:23,426 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-24T04:22:23,426 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/partitions_df27df35-7e55-4e5f-a35c-90471f028318 2024-11-24T04:22:23,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741896_1072 (size=192) 2024-11-24T04:22:23,854 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:23,855 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:23,855 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,796 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-17597476933935832883.jar 2024-11-24T04:22:24,796 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,796 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,798 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,798 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,798 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:22:24,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T04:22:24,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T04:22:24,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T04:22:24,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T04:22:24,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T04:22:24,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T04:22:24,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T04:22:24,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T04:22:24,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T04:22:24,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T04:22:24,802 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,802 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,802 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,803 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,803 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,803 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:24,804 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:24,804 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:test-17324221039661,default:test-1732422103966 2024-11-24T04:22:24,804 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:22:24,804 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:154) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:22:24,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:24,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:24,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:22:24,805 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:22:25,676 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-9331560826517997709.jar 2024-11-24T04:22:25,685 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:22:25,710 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422122082 for WAL files 2024-11-24T04:22:25,711 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422122082 2024-11-24T04:22:25,711 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422122126 for WAL files 2024-11-24T04:22:25,712 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422122126 2024-11-24T04:22:25,804 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local1495211820_0003_90d28037-bf05-420b-9c59-7560c6e377b0/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-24T04:22:25,804 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local1495211820_0003_90d28037-bf05-420b-9c59-7560c6e377b0/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-24T04:22:25,886 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422122126 for hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422122126 (-9223372036854775808:9223372036854775807) length:30490 2024-11-24T04:22:25,906 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-24T04:22:25,907 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-24T04:22:25,944 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422122082 for hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422122082 (-9223372036854775808:9223372036854775807) length:1198 2024-11-24T04:22:25,948 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-24T04:22:25,949 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-24T04:22:26,021 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bfc059a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:26,021 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:22:26,022 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:22:26,022 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:22:26,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:22:26,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:22:26,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2999f339, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:26,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:22:26,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:22:26,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:26,024 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:22:26,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12d975d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:26,026 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:22:26,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:22:26,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:22:26,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:22:26,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17324221039661', row='row-t20', locateType=CURRENT is [region=ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:22:26,030 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-24T04:22:26,031 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:22:26,031 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:26,031 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:26,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:22:26,031 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-11-24T04:22:26,031 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: e57b7116ff5e 2024-11-24T04:22:26,036 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/backupUT/.tmp/backup_1732422137253/_temporary/0/_temporary/attempt_local1495211820_0003_r_000000_0/ns2/test-17324221039661/f/7d95b34f338145ab876d96b003a35700 is 38, key is row-t20/f:q1/1732422135882/Put/seqid=0 2024-11-24T04:22:26,038 WARN [Thread-950 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [e57b7116ff5e:36347] Specified favored nodes: [e57b7116ff5e:36347] 2024-11-24T04:22:26,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741897_1073 (size=5275) 2024-11-24T04:22:26,069 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11a2ea18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:26,069 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:22:26,069 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:22:26,070 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:22:26,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:22:26,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:22:26,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cd3d421, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:26,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:22:26,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:22:26,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:26,072 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:22:26,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76bce323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:26,073 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:22:26,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:22:26,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:22:26,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:22:26,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732422103966', row='row-test-1732422103966-1-0', locateType=CURRENT is [region=test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594., hostname=e57b7116ff5e,36347,1732422095008, seqNum=107] 2024-11-24T04:22:26,077 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-24T04:22:26,077 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:22:26,077 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:26,077 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:26,077 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:22:26,077 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732422103966-1-0] 2024-11-24T04:22:26,077 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: e57b7116ff5e 2024-11-24T04:22:26,089 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ed47a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:26,089 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:22:26,089 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:22:26,090 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:22:26,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:22:26,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:22:26,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a7f8cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:26,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:22:26,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:22:26,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:26,092 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:22:26,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3476856c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:26,093 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:22:26,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:22:26,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:22:26,095 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:22:26,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732422103966', row='row-test-1732422103966-3-0', locateType=CURRENT is [region=test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594., hostname=e57b7116ff5e,36347,1732422095008, seqNum=107] 2024-11-24T04:22:26,097 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-24T04:22:26,097 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:22:26,097 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:26,097 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:26,097 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:22:26,098 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732422103966-3-0] 2024-11-24T04:22:26,098 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: e57b7116ff5e 2024-11-24T04:22:26,107 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/backupUT/.tmp/backup_1732422137253/_temporary/0/_temporary/attempt_local1495211820_0003_r_000001_0/default/test-1732422103966/f/e9f2da723d9a46a8b185c6653a034793 is 59, key is row-test-1732422103966-1-10/f:q1/1732422135476/Put/seqid=0 2024-11-24T04:22:26,109 WARN [Thread-962 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [e57b7116ff5e:36347] Specified favored nodes: [e57b7116ff5e:36347] 2024-11-24T04:22:26,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741898_1074 (size=10770) 2024-11-24T04:22:26,514 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/backupUT/.tmp/backup_1732422137253/_temporary/0/_temporary/attempt_local1495211820_0003_r_000001_0/default/test-1732422103966/mob/048d527910fc4e9c9a6658ada9728c6c is 63, key is row-test-1732422103966-3-100/mob:q1/1732422135842/Put/seqid=0 2024-11-24T04:22:26,517 WARN [Thread-964 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [e57b7116ff5e:36347] Specified favored nodes: [e57b7116ff5e:36347] 2024-11-24T04:22:26,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741899_1075 (size=11811) 2024-11-24T04:22:27,849 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:45347/backupUT 2024-11-24T04:22:27,849 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1732422137253 2024-11-24T04:22:27,849 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-11-24T04:22:27,865 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:45347/backupUT/.tmp/backup_1732422137253, hdfs://localhost:45347/backupUT] 2024-11-24T04:22:27,935 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:22:28,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741900_1076 (size=5275) 2024-11-24T04:22:28,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741901_1077 (size=11811) 2024-11-24T04:22:28,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741902_1078 (size=10770) 2024-11-24T04:22:28,951 INFO [master/e57b7116ff5e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-24T04:22:28,951 INFO [master/e57b7116ff5e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-24T04:22:29,538 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-11-24T04:22:29,544 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1732422137253 2024-11-24T04:22:29,545 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 27856 bytes copied." 2024-11-24T04:22:29,546 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local1992762842_0004 completed: true true 2024-11-24T04:22:29,548 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=3825260 FILE: Number of bytes written=6211350 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=231467 HDFS: Number of bytes written=2433185 HDFS: Number of read operations=643 HDFS: Number of large read operations=0 HDFS: Number of write operations=250 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=12 Map output records=0 Input split bytes=340 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=3004 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=27856 Bytes Copied=27856 Bytes Expected=27856 Files Copied=4 DIR_COPY=8 2024-11-24T04:22:29,548 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:45347/backupUT/.tmp/backup_1732422137253 to hdfs://localhost:45347/backupUT finished. 2024-11-24T04:22:29,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system_bulk', row='bulk:ns2:test-17324221039661:', locateType=CURRENT is [region=backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:22:29,564 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732422137253 2024-11-24T04:22:29,567 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732422115214} 2024-11-24T04:22:29,568 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-24T04:22:29,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741903_1079 (size=384) 2024-11-24T04:22:29,974 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45347/backupUT/backup_1732422137253/.backup.manifest 2024-11-24T04:22:29,974 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732422137253 finished: type=INCREMENTAL,tablelist=ns2:test-17324221039661;test-1732422103966,targetRootDir=hdfs://localhost:45347/backupUT,startts=1732422141533,completets=1732422149564,bytescopied=0 2024-11-24T04:22:29,974 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-24T04:22:29,979 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-11-24T04:22:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-24T04:22:29,983 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-24T04:22:29,985 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-24T04:22:29,988 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732422137253 completed. 2024-11-24T04:22:29,988 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-11-24T04:22:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:22:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:22:30,007 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45347/backupUT/backup_1732422137253/.backup.manifest 2024-11-24T04:22:30,010 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732422137253/.backup.manifest 2024-11-24T04:22:30,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table test-1732422103966 from 'test-1732422103966', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1732422103966', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:22:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1732422103966 2024-11-24T04:22:30,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741904_1080 (size=908) 2024-11-24T04:22:30,431 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/.tabledesc/.tableinfo.0000000003.950 2024-11-24T04:22:30,432 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/.tabledesc/.tableinfo.0000000004.908 2024-11-24T04:22:30,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1732422103966}] 2024-11-24T04:22:30,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, REOPEN/MOVE}] 2024-11-24T04:22:30,441 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, REOPEN/MOVE 2024-11-24T04:22:30,441 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=2c375e169f1afc2d9cce55928a15d594, regionState=CLOSING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:30,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, REOPEN/MOVE because future has completed 2024-11-24T04:22:30,443 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T04:22:30,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2c375e169f1afc2d9cce55928a15d594, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:22:30,596 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(122): Close 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:30,596 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T04:22:30,596 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1722): Closing 2c375e169f1afc2d9cce55928a15d594, disabling compactions & flushes 2024-11-24T04:22:30,596 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1755): Closing region test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:22:30,596 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:22:30,596 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1843): Acquired close lock on test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. after waiting 0 ms 2024-11-24T04:22:30,596 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1853): Updates disabled for region test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:22:30,602 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=106 2024-11-24T04:22:30,602 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:22:30,602 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1973): Closed test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:22:30,602 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1676): Region close journal for 2c375e169f1afc2d9cce55928a15d594: Waiting for close lock at 1732422150596Running coprocessor pre-close hooks at 1732422150596Disabling compacts and flushes for region at 1732422150596Disabling writes for close at 1732422150596Writing region close event to WAL at 1732422150598 (+2 ms)Running coprocessor post-close hooks at 1732422150602 (+4 ms)Closed at 1732422150602 2024-11-24T04:22:30,603 WARN [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegionServer(3299): Not adding moved region record: 2c375e169f1afc2d9cce55928a15d594 to self. 2024-11-24T04:22:30,605 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(157): Closed 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:30,605 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=2c375e169f1afc2d9cce55928a15d594, regionState=CLOSED 2024-11-24T04:22:30,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2c375e169f1afc2d9cce55928a15d594, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:22:30,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-24T04:22:30,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseRegionProcedure 2c375e169f1afc2d9cce55928a15d594, server=e57b7116ff5e,36347,1732422095008 in 164 msec 2024-11-24T04:22:30,611 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, REOPEN/MOVE; state=CLOSED, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=true 2024-11-24T04:22:30,762 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=2c375e169f1afc2d9cce55928a15d594, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:30,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, REOPEN/MOVE because future has completed 2024-11-24T04:22:30,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c375e169f1afc2d9cce55928a15d594, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:22:30,927 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:22:30,927 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 2c375e169f1afc2d9cce55928a15d594, NAME => 'test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:22:30,928 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:22:30,928 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732422103966 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:30,928 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:22:30,928 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:30,928 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:30,930 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:30,932 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c375e169f1afc2d9cce55928a15d594 columnFamilyName f 2024-11-24T04:22:30,932 DEBUG [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:22:30,941 DEBUG [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/60684cdd8d5e45cbaabfe6505beaad9f 2024-11-24T04:22:30,947 DEBUG [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3 2024-11-24T04:22:30,948 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.HStore(327): Store=2c375e169f1afc2d9cce55928a15d594/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:22:30,948 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:30,949 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c375e169f1afc2d9cce55928a15d594 columnFamilyName f2 2024-11-24T04:22:30,949 DEBUG [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:22:30,950 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.HStore(327): Store=2c375e169f1afc2d9cce55928a15d594/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:22:30,950 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:30,951 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c375e169f1afc2d9cce55928a15d594 columnFamilyName mob 2024-11-24T04:22:30,952 DEBUG [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:22:30,959 DEBUG [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/4adb0dbc87f947bdaad87e7746c93da8 2024-11-24T04:22:30,963 DEBUG [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/682085848b1a4dc1a2b01cad48fbef3b 2024-11-24T04:22:30,963 INFO [StoreOpener-2c375e169f1afc2d9cce55928a15d594-1 {}] regionserver.HStore(327): Store=2c375e169f1afc2d9cce55928a15d594/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:22:30,964 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:30,965 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:30,966 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:30,967 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:30,967 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:30,967 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1732422103966 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:22:30,968 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:30,969 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 2c375e169f1afc2d9cce55928a15d594; next sequenceid=338; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9666713120, jitterRate=-0.09971718490123749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:22:30,969 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:30,972 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 2c375e169f1afc2d9cce55928a15d594: Running coprocessor pre-open hook at 1732422150928Writing region info on filesystem at 1732422150928Initializing all the Stores at 1732422150930 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422150930Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422150930Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422150930Cleaning up temporary data from old regions at 1732422150967 (+37 ms)Running coprocessor post-open hooks at 1732422150969 (+2 ms)Region opened successfully at 1732422150972 (+3 ms) 2024-11-24T04:22:30,973 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594., pid=46, masterSystemTime=1732422150922 2024-11-24T04:22:30,975 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:22:30,975 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:22:30,975 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=2c375e169f1afc2d9cce55928a15d594, regionState=OPEN, openSeqNum=338, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:30,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c375e169f1afc2d9cce55928a15d594, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:22:30,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=44 2024-11-24T04:22:30,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=44, state=SUCCESS, hasLock=false; OpenRegionProcedure 2c375e169f1afc2d9cce55928a15d594, server=e57b7116ff5e,36347,1732422095008 in 211 msec 2024-11-24T04:22:30,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-24T04:22:30,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732422103966, region=2c375e169f1afc2d9cce55928a15d594, REOPEN/MOVE in 540 msec 2024-11-24T04:22:30,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-24T04:22:30,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1732422103966 in 545 msec 2024-11-24T04:22:30,986 DEBUG [PEWorker-3 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=test-1732422103966 2024-11-24T04:22:30,993 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f3/f9440cbd308f4fa3b08696b88b8a8b86 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f3/f9440cbd308f4fa3b08696b88b8a8b86 2024-11-24T04:22:30,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1732422103966 in 982 msec 2024-11-24T04:22:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T04:22:30,997 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1732422103966 completed 2024-11-24T04:22:31,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:22:31,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:22:31,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:22:31,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:22:31,033 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-24T04:22:31,037 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732422150998 2024-11-24T04:22:31,039 DEBUG [Time-limited test {}] impl.TableBackupClient(330): Dependent incremental backup image: {BackupID=backup_1732422137253} 2024-11-24T04:22:31,039 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732422115214} 2024-11-24T04:22:31,039 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 2 ancestors for the current backup. 2024-11-24T04:22:31,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:22:31,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:22:31,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:22:31,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:22:31,082 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected Exception : Mismatch in column family descriptor for table: test-1732422103966 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException: Mismatch in column family descriptor for table: test-1732422103966 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException$ColumnFamilyMismatchExceptionBuilder.build(ColumnFamilyMismatchException.java:56) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.verifyCfCompatibility(IncrementalTableBackupClient.java:513) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:280) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.lambda$TestIncBackupRestore$0(TestIncrementalBackup.java:171) ~[test-classes/:?] at org.junit.Assert.assertThrows(Assert.java:1001) ~[junit-4.13.2.jar:4.13.2] at org.junit.Assert.assertThrows(Assert.java:981) ~[junit-4.13.2.jar:4.13.2] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:170) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:22:31,085 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1732422150998,startts=0,failedts=1732422151083,failedphase=REQUEST,failedmessage= Mismatch in column family descriptor for table: test-1732422103966 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'tru 2024-11-24T04:22:31,085 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-24T04:22:31,088 WARN [Time-limited test {}] impl.BackupSystemTable(1549): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2024-11-24T04:22:31,088 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-24T04:22:31,090 ERROR [Time-limited test {}] impl.BackupSystemTable(1578): Snapshot snapshot_backup_system does not exists 2024-11-24T04:22:31,090 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: REQUEST 2024-11-24T04:22:31,092 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-24T04:22:31,094 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1732422150998 failed. 2024-11-24T04:22:31,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:22:31,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:22:31,111 INFO [Time-limited test {}] impl.BackupManager(225): Full backup all the tables available in the cluster: [backup:system_bulk, test-1732422103966, ns2:test-17324221039661, ns3:test-17324221039662, ns4:test-17324221039663] 2024-11-24T04:22:31,112 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-24T04:22:31,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T04:22:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732422151116 (current time:1732422151116). 2024-11-24T04:22:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T04:22:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-24T04:22:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b873b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:22:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:22:31,118 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:22:31,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:22:31,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:22:31,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@487cf4f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:31,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:22:31,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:22:31,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:31,119 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:22:31,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f70f342, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:22:31,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:22:31,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:22:31,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:22:31,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719. 2024-11-24T04:22:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:22:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:31,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:22:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T04:22:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T04:22:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-24T04:22:31,126 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T04:22:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-24T04:22:31,127 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T04:22:31,129 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T04:22:31,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741905_1081 (size=52) 2024-11-24T04:22:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-24T04:22:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-24T04:22:31,536 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T04:22:31,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2}] 2024-11-24T04:22:31,537 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:22:31,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36347 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=48 2024-11-24T04:22:31,691 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:22:31,692 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2902): Flushing 5a030641752b7def7c38c5f6295501b2 2/2 column families, dataSize=2.61 KB heapSize=3.99 KB 2024-11-24T04:22:31,702 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/meta/09622658e56b4cb896386e23d2791852 is 158, key is trslm:hdfs://localhost:45347/backupUT\x00ns2:test-17324221039661/meta:log-roll-map/1732422149553/Put/seqid=0 2024-11-24T04:22:31,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741906_1082 (size=5745) 2024-11-24T04:22:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-24T04:22:32,107 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=492 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/meta/09622658e56b4cb896386e23d2791852 2024-11-24T04:22:32,123 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/session/c3f9856af53640e1aa92faceb2b4a6bd is 1245, key is session:backup_1732422150998/session:context/1732422151091/Put/seqid=0 2024-11-24T04:22:32,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741907_1083 (size=6682) 2024-11-24T04:22:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-24T04:22:32,380 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T04:22:32,529 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/session/c3f9856af53640e1aa92faceb2b4a6bd 2024-11-24T04:22:32,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/meta/09622658e56b4cb896386e23d2791852 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/09622658e56b4cb896386e23d2791852 2024-11-24T04:22:32,551 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/09622658e56b4cb896386e23d2791852, entries=4, sequenceid=31, filesize=5.6 K 2024-11-24T04:22:32,553 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/session/c3f9856af53640e1aa92faceb2b4a6bd as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/c3f9856af53640e1aa92faceb2b4a6bd 2024-11-24T04:22:32,560 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/c3f9856af53640e1aa92faceb2b4a6bd, entries=3, sequenceid=31, filesize=6.5 K 2024-11-24T04:22:32,561 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(3140): Finished flush of dataSize ~2.61 KB/2669, heapSize ~3.96 KB/4056, currentSize=0 B/0 for 5a030641752b7def7c38c5f6295501b2 in 870ms, sequenceid=31, compaction requested=true 2024-11-24T04:22:32,561 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2603): Flush status journal for 5a030641752b7def7c38c5f6295501b2: 2024-11-24T04:22:32,561 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. for snapshot_backup_system completed. 2024-11-24T04:22:32,561 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2.' region-info for snapshot=snapshot_backup_system 2024-11-24T04:22:32,561 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T04:22:32,561 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/d61b2831ca274cc7989a43ee1404fb09, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/09622658e56b4cb896386e23d2791852] hfiles 2024-11-24T04:22:32,561 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/d61b2831ca274cc7989a43ee1404fb09 for snapshot=snapshot_backup_system 2024-11-24T04:22:32,561 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/09622658e56b4cb896386e23d2791852 for snapshot=snapshot_backup_system 2024-11-24T04:22:32,561 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/e44333cb30954234bdc3a05f162b96cd, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/fea54c00fd9144c2b57bfc7379644a05, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/c3f9856af53640e1aa92faceb2b4a6bd] hfiles 2024-11-24T04:22:32,561 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/e44333cb30954234bdc3a05f162b96cd for snapshot=snapshot_backup_system 2024-11-24T04:22:32,561 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/fea54c00fd9144c2b57bfc7379644a05 for snapshot=snapshot_backup_system 2024-11-24T04:22:32,561 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/c3f9856af53640e1aa92faceb2b4a6bd for snapshot=snapshot_backup_system 2024-11-24T04:22:32,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741908_1084 (size=251) 2024-11-24T04:22:32,967 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:22:32,968 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-11-24T04:22:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster(4169): Remote procedure done, pid=48 2024-11-24T04:22:32,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:22:32,969 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:22:32,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-11-24T04:22:32,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2 in 1.4350 sec 2024-11-24T04:22:32,974 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T04:22:32,976 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T04:22:32,977 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T04:22:32,977 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-24T04:22:32,979 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-24T04:22:32,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741909_1085 (size=888) 2024-11-24T04:22:33,027 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region d3f34500755154053262a8d9790ac0a1, had cached 0 bytes from a total of 8413 2024-11-24T04:22:33,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-24T04:22:33,397 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T04:22:33,408 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T04:22:33,409 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_backup_system 2024-11-24T04:22:33,411 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T04:22:33,411 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-24T04:22:33,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2870 sec 2024-11-24T04:22:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-24T04:22:35,292 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-24T04:22:35,292 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732422151095 started at 1732422155292. 2024-11-24T04:22:35,297 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732422151095 has been started. 2024-11-24T04:22:35,308 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-24T04:22:35,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-11-24T04:22:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-24T04:22:35,309 INFO [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-24T04:22:35,309 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-24T04:22:35,309 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-24T04:22:35,309 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-24T04:22:35,310 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T04:22:35,310 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T04:22:35,342 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T04:22:35,342 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T04:22:35,343 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T04:22:35,343 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,343 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-24T04:22:35,344 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-24T04:22:35,345 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T04:22:35,346 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-24T04:22:35,346 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T04:22:35,346 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-24T04:22:35,346 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-24T04:22:35,346 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-24T04:22:35,347 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-24T04:22:35,347 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-24T04:22:35,347 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-24T04:22:35,347 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-24T04:22:35,347 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'e57b7116ff5e,36347,1732422095008' joining acquired barrier for procedure (rolllog) in zk 2024-11-24T04:22:35,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,353 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:35,353 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,353 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T04:22:35,353 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T04:22:35,354 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-24T04:22:35,354 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-24T04:22:35,354 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T04:22:35,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T04:22:35,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:35,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,356 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T04:22:35,356 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'e57b7116ff5e,36347,1732422095008' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-24T04:22:35,356 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@33f3acb2[Count = 0] remaining members to acquire global barrier 2024-11-24T04:22:35,357 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-24T04:22:35,357 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:35,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:35,363 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:35,363 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:35,363 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-24T04:22:35,363 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,363 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-24T04:22:35,363 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-24T04:22:35,363 DEBUG [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,363 INFO [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732422141596 highest: 1732422141631 on e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,364 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0:(num 1732422141596) roll requested 2024-11-24T04:22:35,379 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422155364, exclude list is [], retry=0 2024-11-24T04:22:35,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:22:35,383 INFO [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422141596 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422155364 2024-11-24T04:22:35,383 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:22:35,383 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422141596 is not closed yet, will try archiving it next time 2024-11-24T04:22:35,383 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL e57b7116ff5e%2C36347%2C1732422095008.meta:.meta(num 1732422141614) roll requested 2024-11-24T04:22:35,384 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422141596 not finished, retry = 0 2024-11-24T04:22:35,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741889_1065 (size=93) 2024-11-24T04:22:35,395 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422155383.meta, exclude list is [], retry=0 2024-11-24T04:22:35,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:22:35,399 INFO [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422141614.meta with entries=4, filesize=1.85 KB; new WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422155383.meta 2024-11-24T04:22:35,400 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:22:35,400 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422141614.meta is not closed yet, will try archiving it next time 2024-11-24T04:22:35,400 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1:(num 1732422141631) roll requested 2024-11-24T04:22:35,401 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422141614.meta not finished, retry = 0 2024-11-24T04:22:35,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741890_1066 (size=1903) 2024-11-24T04:22:35,411 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422155400, exclude list is [], retry=0 2024-11-24T04:22:35,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:22:35,415 INFO [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422141631 with entries=15, filesize=4.80 KB; new WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422155400 2024-11-24T04:22:35,415 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:22:35,415 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422141631 is not closed yet, will try archiving it next time 2024-11-24T04:22:35,415 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422122126 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422122126 2024-11-24T04:22:35,416 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422141631 not finished, retry = 0 2024-11-24T04:22:35,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741891_1067 (size=4927) 2024-11-24T04:22:35,424 DEBUG [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-11-24T04:22:35,424 INFO [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732422155364 on e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:22:35,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:22:35,444 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-24T04:22:35,444 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'e57b7116ff5e,36347,1732422095008' in zk 2024-11-24T04:22:35,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,457 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-24T04:22:35,457 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T04:22:35,457 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,457 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-24T04:22:35,457 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T04:22:35,457 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T04:22:35,458 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T04:22:35,459 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T04:22:35,459 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:35,460 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,460 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T04:22:35,461 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:35,461 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,462 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'e57b7116ff5e,36347,1732422095008': 2024-11-24T04:22:35,463 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'e57b7116ff5e,36347,1732422095008' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-24T04:22:35,463 INFO [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-24T04:22:35,463 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-24T04:22:35,463 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-24T04:22:35,463 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-24T04:22:35,463 INFO [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-24T04:22:35,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T04:22:35,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T04:22:35,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T04:22:35,468 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-24T04:22:35,468 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-24T04:22:35,468 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T04:22:35,468 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T04:22:35,468 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T04:22:35,469 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T04:22:35,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T04:22:35,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T04:22:35,469 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T04:22:35,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:35,470 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T04:22:35,471 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:35,471 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,471 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,471 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T04:22:35,472 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:22:35,472 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T04:22:35,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T04:22:35,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T04:22:35,479 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T04:22:35,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:22:35,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T04:22:35,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:35,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T04:22:35,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T04:22:35,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T04:22:35,479 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T04:22:35,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-24T04:22:35,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-24T04:22:35,479 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T04:22:35,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T04:22:35,485 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422141596 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422141596 2024-11-24T04:22:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-24T04:22:35,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1732422155493_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-24T04:22:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732422155493 (current time:1732422155493). 2024-11-24T04:22:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T04:22:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732422155493_backup_system_bulk VERSION not specified, setting to 2 2024-11-24T04:22:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@254cc8d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:22:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:22:35,495 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:22:35,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:22:35,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:22:35,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38794bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:35,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:22:35,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:22:35,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:35,496 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:22:35,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e8cba6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:22:35,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:22:35,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:22:35,499 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:22:35,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719. 2024-11-24T04:22:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:22:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:35,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:22:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T04:22:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=snapshot_1732422155493_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-24T04:22:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732422155493_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 49 2024-11-24T04:22:35,503 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=snapshot_1732422155493_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T04:22:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-24T04:22:35,504 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=snapshot_1732422155493_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T04:22:35,507 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=snapshot_1732422155493_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T04:22:35,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741913_1089 (size=76) 2024-11-24T04:22:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-24T04:22:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-24T04:22:35,915 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=snapshot_1732422155493_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T04:22:35,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4317ba5909dca438c339af86075fb75a}] 2024-11-24T04:22:35,918 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4317ba5909dca438c339af86075fb75a 2024-11-24T04:22:36,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36347 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=50 2024-11-24T04:22:36,072 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=50}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. 2024-11-24T04:22:36,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=50}] regionserver.HRegion(2603): Flush status journal for 4317ba5909dca438c339af86075fb75a: 2024-11-24T04:22:36,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=50}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. for snapshot_1732422155493_backup_system_bulk completed. 2024-11-24T04:22:36,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=50}] snapshot.SnapshotManifest(241): Storing 'backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a.' region-info for snapshot=snapshot_1732422155493_backup_system_bulk 2024-11-24T04:22:36,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=50}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T04:22:36,074 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=50}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-24T04:22:36,074 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=50}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-24T04:22:36,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741914_1090 (size=61) 2024-11-24T04:22:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-24T04:22:36,134 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-24T04:22:36,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=50}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. 2024-11-24T04:22:36,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=50 2024-11-24T04:22:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster(4169): Remote procedure done, pid=50 2024-11-24T04:22:36,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732422155493_backup_system_bulk on region 4317ba5909dca438c339af86075fb75a 2024-11-24T04:22:36,485 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4317ba5909dca438c339af86075fb75a 2024-11-24T04:22:36,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-11-24T04:22:36,488 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=snapshot_1732422155493_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T04:22:36,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 4317ba5909dca438c339af86075fb75a in 570 msec 2024-11-24T04:22:36,488 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=snapshot_1732422155493_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T04:22:36,489 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=snapshot_1732422155493_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T04:22:36,489 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732422155493_backup_system_bulk 2024-11-24T04:22:36,490 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_1732422155493_backup_system_bulk 2024-11-24T04:22:36,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741915_1091 (size=702) 2024-11-24T04:22:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-24T04:22:36,904 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=snapshot_1732422155493_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T04:22:36,911 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=snapshot_1732422155493_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T04:22:36,912 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_1732422155493_backup_system_bulk to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_1732422155493_backup_system_bulk 2024-11-24T04:22:36,913 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=snapshot_1732422155493_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T04:22:36,913 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732422155493_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 49 2024-11-24T04:22:36,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=snapshot_1732422155493_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } in 1.4110 sec 2024-11-24T04:22:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-24T04:22:37,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system_bulk completed 2024-11-24T04:22:37,654 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1732422157652_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } 2024-11-24T04:22:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732422157654 (current time:1732422157654). 2024-11-24T04:22:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T04:22:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732422157652_ns2_test-17324221039661 VERSION not specified, setting to 2 2024-11-24T04:22:37,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44e7f53a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:37,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:22:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:22:37,659 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:22:37,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:22:37,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:22:37,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dea9f7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:37,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:22:37,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:22:37,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:37,662 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:22:37,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b693cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:22:37,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:22:37,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:22:37,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:22:37,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719. 2024-11-24T04:22:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:22:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:37,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:22:37,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T04:22:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=51, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=51, snapshot={ ss=snapshot_1732422157652_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } 2024-11-24T04:22:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732422157652_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 }, snapshot procedure id = 51 2024-11-24T04:22:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-24T04:22:37,671 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=51, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=51, snapshot={ ss=snapshot_1732422157652_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T04:22:37,672 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=51, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=51, snapshot={ ss=snapshot_1732422157652_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T04:22:37,675 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=51, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=51, snapshot={ ss=snapshot_1732422157652_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T04:22:37,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741916_1092 (size=86) 2024-11-24T04:22:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-24T04:22:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-24T04:22:38,084 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=51, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=51, snapshot={ ss=snapshot_1732422157652_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T04:22:38,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d3f34500755154053262a8d9790ac0a1}] 2024-11-24T04:22:38,085 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d3f34500755154053262a8d9790ac0a1 2024-11-24T04:22:38,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36347 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=52 2024-11-24T04:22:38,237 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:22:38,237 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] regionserver.HRegion(2902): Flushing d3f34500755154053262a8d9790ac0a1 1/1 column families, dataSize=170 B heapSize=816 B 2024-11-24T04:22:38,243 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/.tmp/f/1d220a07c36347908f109231fa9cf854 is 38, key is row-t20/f:q1/1732422135882/Put/seqid=0 2024-11-24T04:22:38,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741917_1093 (size=5134) 2024-11-24T04:22:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-24T04:22:38,652 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/.tmp/f/1d220a07c36347908f109231fa9cf854 2024-11-24T04:22:38,666 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/.tmp/f/1d220a07c36347908f109231fa9cf854 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/1d220a07c36347908f109231fa9cf854 2024-11-24T04:22:38,675 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/1d220a07c36347908f109231fa9cf854, entries=5, sequenceid=111, filesize=5.0 K 2024-11-24T04:22:38,676 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] regionserver.HRegion(3140): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for d3f34500755154053262a8d9790ac0a1 in 439ms, sequenceid=111, compaction requested=false 2024-11-24T04:22:38,676 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] regionserver.HRegion(2603): Flush status journal for d3f34500755154053262a8d9790ac0a1: 2024-11-24T04:22:38,676 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. for snapshot_1732422157652_ns2_test-17324221039661 completed. 2024-11-24T04:22:38,676 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1.' region-info for snapshot=snapshot_1732422157652_ns2_test-17324221039661 2024-11-24T04:22:38,677 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T04:22:38,677 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/933ab35b68994dc8be3ccc706f0c0aa9, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/1d220a07c36347908f109231fa9cf854] hfiles 2024-11-24T04:22:38,677 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/933ab35b68994dc8be3ccc706f0c0aa9 for snapshot=snapshot_1732422157652_ns2_test-17324221039661 2024-11-24T04:22:38,677 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/1d220a07c36347908f109231fa9cf854 for snapshot=snapshot_1732422157652_ns2_test-17324221039661 2024-11-24T04:22:38,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741918_1094 (size=130) 2024-11-24T04:22:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-24T04:22:39,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:22:39,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=52}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=52 2024-11-24T04:22:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster(4169): Remote procedure done, pid=52 2024-11-24T04:22:39,086 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732422157652_ns2_test-17324221039661 on region d3f34500755154053262a8d9790ac0a1 2024-11-24T04:22:39,086 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d3f34500755154053262a8d9790ac0a1 2024-11-24T04:22:39,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-24T04:22:39,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; SnapshotRegionProcedure d3f34500755154053262a8d9790ac0a1 in 1.0050 sec 2024-11-24T04:22:39,094 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=51, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=51, snapshot={ ss=snapshot_1732422157652_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T04:22:39,096 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=51, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=51, snapshot={ ss=snapshot_1732422157652_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T04:22:39,097 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=51, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=51, snapshot={ ss=snapshot_1732422157652_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T04:22:39,097 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732422157652_ns2_test-17324221039661 2024-11-24T04:22:39,097 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_1732422157652_ns2_test-17324221039661 2024-11-24T04:22:39,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741919_1095 (size=496) 2024-11-24T04:22:39,510 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=51, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=51, snapshot={ ss=snapshot_1732422157652_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T04:22:39,520 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=51, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=51, snapshot={ ss=snapshot_1732422157652_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T04:22:39,520 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_1732422157652_ns2_test-17324221039661 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_1732422157652_ns2_test-17324221039661 2024-11-24T04:22:39,522 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=51, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=51, snapshot={ ss=snapshot_1732422157652_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T04:22:39,522 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732422157652_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 }, snapshot procedure id = 51 2024-11-24T04:22:39,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=51, snapshot={ ss=snapshot_1732422157652_ns2_test-17324221039661 table=ns2:test-17324221039661 type=FLUSH ttl=0 } in 1.8540 sec 2024-11-24T04:22:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-24T04:22:39,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17324221039661 completed 2024-11-24T04:22:39,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1732422159823_ns4_test-17324221039663 table=ns4:test-17324221039663 type=FLUSH ttl=0 } 2024-11-24T04:22:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732422159825 (current time:1732422159825). 2024-11-24T04:22:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T04:22:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732422159823_ns4_test-17324221039663 VERSION not specified, setting to 2 2024-11-24T04:22:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19c901de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:22:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:22:39,830 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:22:39,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:22:39,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:22:39,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cb95978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:39,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:22:39,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:22:39,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:39,832 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:22:39,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da97c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:22:39,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:22:39,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:22:39,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:22:39,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719. 2024-11-24T04:22:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:22:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:39,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:22:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T04:22:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_1732422159823_ns4_test-17324221039663 table=ns4:test-17324221039663 type=FLUSH ttl=0 } 2024-11-24T04:22:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732422159823_ns4_test-17324221039663 table=ns4:test-17324221039663 type=FLUSH ttl=0 }, snapshot procedure id = 53 2024-11-24T04:22:39,838 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_1732422159823_ns4_test-17324221039663 table=ns4:test-17324221039663 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T04:22:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-24T04:22:39,838 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_1732422159823_ns4_test-17324221039663 table=ns4:test-17324221039663 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T04:22:39,840 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_1732422159823_ns4_test-17324221039663 table=ns4:test-17324221039663 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T04:22:39,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741920_1096 (size=86) 2024-11-24T04:22:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-24T04:22:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-24T04:22:40,250 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_1732422159823_ns4_test-17324221039663 table=ns4:test-17324221039663 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T04:22:40,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a75e6c6417a138961442a072386b6bcc}] 2024-11-24T04:22:40,252 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a75e6c6417a138961442a072386b6bcc 2024-11-24T04:22:40,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36347 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=54 2024-11-24T04:22:40,406 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. 2024-11-24T04:22:40,407 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.HRegion(2603): Flush status journal for a75e6c6417a138961442a072386b6bcc: 2024-11-24T04:22:40,407 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. for snapshot_1732422159823_ns4_test-17324221039663 completed. 2024-11-24T04:22:40,407 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(241): Storing 'ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc.' region-info for snapshot=snapshot_1732422159823_ns4_test-17324221039663 2024-11-24T04:22:40,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T04:22:40,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-24T04:22:40,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-24T04:22:40,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741921_1097 (size=52) 2024-11-24T04:22:40,873 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. 2024-11-24T04:22:40,873 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=54 2024-11-24T04:22:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster(4169): Remote procedure done, pid=54 2024-11-24T04:22:40,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732422159823_ns4_test-17324221039663 on region a75e6c6417a138961442a072386b6bcc 2024-11-24T04:22:40,874 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a75e6c6417a138961442a072386b6bcc 2024-11-24T04:22:40,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-24T04:22:40,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; SnapshotRegionProcedure a75e6c6417a138961442a072386b6bcc in 625 msec 2024-11-24T04:22:40,879 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_1732422159823_ns4_test-17324221039663 table=ns4:test-17324221039663 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T04:22:40,880 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_1732422159823_ns4_test-17324221039663 table=ns4:test-17324221039663 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T04:22:40,881 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_1732422159823_ns4_test-17324221039663 table=ns4:test-17324221039663 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T04:22:40,881 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732422159823_ns4_test-17324221039663 2024-11-24T04:22:40,882 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_1732422159823_ns4_test-17324221039663 2024-11-24T04:22:40,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741922_1098 (size=443) 2024-11-24T04:22:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-24T04:22:41,294 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_1732422159823_ns4_test-17324221039663 table=ns4:test-17324221039663 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T04:22:41,299 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_1732422159823_ns4_test-17324221039663 table=ns4:test-17324221039663 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T04:22:41,300 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_1732422159823_ns4_test-17324221039663 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_1732422159823_ns4_test-17324221039663 2024-11-24T04:22:41,302 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_1732422159823_ns4_test-17324221039663 table=ns4:test-17324221039663 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T04:22:41,302 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732422159823_ns4_test-17324221039663 table=ns4:test-17324221039663 type=FLUSH ttl=0 }, snapshot procedure id = 53 2024-11-24T04:22:41,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_1732422159823_ns4_test-17324221039663 table=ns4:test-17324221039663 type=FLUSH ttl=0 } in 1.4660 sec 2024-11-24T04:22:41,547 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5a030641752b7def7c38c5f6295501b2, had cached 0 bytes from a total of 28925 2024-11-24T04:22:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-24T04:22:41,983 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns4:test-17324221039663 completed 2024-11-24T04:22:41,985 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1732422161983_ns3_test-17324221039662 table=ns3:test-17324221039662 type=FLUSH ttl=0 } 2024-11-24T04:22:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732422161985 (current time:1732422161985). 2024-11-24T04:22:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T04:22:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732422161983_ns3_test-17324221039662 VERSION not specified, setting to 2 2024-11-24T04:22:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56591963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:22:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:22:41,988 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:22:41,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:22:41,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:22:41,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7b5623, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:41,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:22:41,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:22:41,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:41,990 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:22:41,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@399fb31e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:22:41,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:22:41,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:22:41,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:22:41,993 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719. 2024-11-24T04:22:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:22:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:41,993 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:22:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T04:22:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=55, snapshot={ ss=snapshot_1732422161983_ns3_test-17324221039662 table=ns3:test-17324221039662 type=FLUSH ttl=0 } 2024-11-24T04:22:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732422161983_ns3_test-17324221039662 table=ns3:test-17324221039662 type=FLUSH ttl=0 }, snapshot procedure id = 55 2024-11-24T04:22:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-24T04:22:41,996 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=55, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=55, snapshot={ ss=snapshot_1732422161983_ns3_test-17324221039662 table=ns3:test-17324221039662 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T04:22:41,997 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=55, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=55, snapshot={ ss=snapshot_1732422161983_ns3_test-17324221039662 table=ns3:test-17324221039662 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T04:22:41,999 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=55, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=55, snapshot={ ss=snapshot_1732422161983_ns3_test-17324221039662 table=ns3:test-17324221039662 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T04:22:42,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741923_1099 (size=86) 2024-11-24T04:22:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-24T04:22:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-24T04:22:42,408 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=55, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=55, snapshot={ ss=snapshot_1732422161983_ns3_test-17324221039662 table=ns3:test-17324221039662 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T04:22:42,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a35f3acd735a5c01c6819f7a642d051d}] 2024-11-24T04:22:42,411 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a35f3acd735a5c01c6819f7a642d051d 2024-11-24T04:22:42,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36347 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=56 2024-11-24T04:22:42,566 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=56}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. 2024-11-24T04:22:42,566 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=56}] regionserver.HRegion(2603): Flush status journal for a35f3acd735a5c01c6819f7a642d051d: 2024-11-24T04:22:42,567 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=56}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. for snapshot_1732422161983_ns3_test-17324221039662 completed. 2024-11-24T04:22:42,567 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=56}] snapshot.SnapshotManifest(241): Storing 'ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d.' region-info for snapshot=snapshot_1732422161983_ns3_test-17324221039662 2024-11-24T04:22:42,567 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=56}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T04:22:42,567 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=56}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-24T04:22:42,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741924_1100 (size=52) 2024-11-24T04:22:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-24T04:22:42,981 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=56}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. 2024-11-24T04:22:42,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=56 2024-11-24T04:22:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster(4169): Remote procedure done, pid=56 2024-11-24T04:22:42,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732422161983_ns3_test-17324221039662 on region a35f3acd735a5c01c6819f7a642d051d 2024-11-24T04:22:42,984 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a35f3acd735a5c01c6819f7a642d051d 2024-11-24T04:22:42,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-24T04:22:42,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; SnapshotRegionProcedure a35f3acd735a5c01c6819f7a642d051d in 578 msec 2024-11-24T04:22:42,991 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=55, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=55, snapshot={ ss=snapshot_1732422161983_ns3_test-17324221039662 table=ns3:test-17324221039662 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T04:22:42,992 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=55, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=55, snapshot={ ss=snapshot_1732422161983_ns3_test-17324221039662 table=ns3:test-17324221039662 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T04:22:42,992 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=55, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=55, snapshot={ ss=snapshot_1732422161983_ns3_test-17324221039662 table=ns3:test-17324221039662 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T04:22:42,992 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732422161983_ns3_test-17324221039662 2024-11-24T04:22:42,993 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_1732422161983_ns3_test-17324221039662 2024-11-24T04:22:43,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741925_1101 (size=443) 2024-11-24T04:22:43,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-24T04:22:43,409 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=55, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=55, snapshot={ ss=snapshot_1732422161983_ns3_test-17324221039662 table=ns3:test-17324221039662 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T04:22:43,417 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=55, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=55, snapshot={ ss=snapshot_1732422161983_ns3_test-17324221039662 table=ns3:test-17324221039662 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T04:22:43,418 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_1732422161983_ns3_test-17324221039662 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_1732422161983_ns3_test-17324221039662 2024-11-24T04:22:43,420 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=55, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=55, snapshot={ ss=snapshot_1732422161983_ns3_test-17324221039662 table=ns3:test-17324221039662 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T04:22:43,420 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732422161983_ns3_test-17324221039662 table=ns3:test-17324221039662 type=FLUSH ttl=0 }, snapshot procedure id = 55 2024-11-24T04:22:43,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=55, snapshot={ ss=snapshot_1732422161983_ns3_test-17324221039662 table=ns3:test-17324221039662 type=FLUSH ttl=0 } in 1.4260 sec 2024-11-24T04:22:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-24T04:22:44,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns3:test-17324221039662 completed 2024-11-24T04:22:44,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1732422164152_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } 2024-11-24T04:22:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732422164154 (current time:1732422164154). 2024-11-24T04:22:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T04:22:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732422164152_default_test-1732422103966 VERSION not specified, setting to 2 2024-11-24T04:22:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ed9e97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:22:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:22:44,159 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:22:44,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:22:44,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:22:44,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51c8b9c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:44,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:22:44,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:22:44,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:44,161 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:22:44,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd66fbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:22:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:22:44,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:22:44,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:22:44,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:22:44,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719. 2024-11-24T04:22:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:22:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:22:44,165 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:22:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T04:22:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=snapshot_1732422164152_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } 2024-11-24T04:22:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732422164152_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 }, snapshot procedure id = 57 2024-11-24T04:22:44,168 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=snapshot_1732422164152_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T04:22:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T04:22:44,169 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=snapshot_1732422164152_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T04:22:44,171 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=snapshot_1732422164152_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T04:22:44,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741926_1102 (size=84) 2024-11-24T04:22:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T04:22:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T04:22:44,581 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=snapshot_1732422164152_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T04:22:44,581 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 2c375e169f1afc2d9cce55928a15d594}] 2024-11-24T04:22:44,585 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:44,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36347 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=58 2024-11-24T04:22:44,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=58}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:22:44,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=58}] regionserver.HRegion(2603): Flush status journal for 2c375e169f1afc2d9cce55928a15d594: 2024-11-24T04:22:44,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=58}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. for snapshot_1732422164152_default_test-1732422103966 completed. 2024-11-24T04:22:44,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=58}] snapshot.SnapshotManifest(241): Storing 'test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594.' region-info for snapshot=snapshot_1732422164152_default_test-1732422103966 2024-11-24T04:22:44,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=58}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T04:22:44,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=58}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/60684cdd8d5e45cbaabfe6505beaad9f] hfiles 2024-11-24T04:22:44,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=58}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3 for snapshot=snapshot_1732422164152_default_test-1732422103966 2024-11-24T04:22:44,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=58}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/60684cdd8d5e45cbaabfe6505beaad9f for snapshot=snapshot_1732422164152_default_test-1732422103966 2024-11-24T04:22:44,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=58}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-24T04:22:44,740 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=58}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/682085848b1a4dc1a2b01cad48fbef3b, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/4adb0dbc87f947bdaad87e7746c93da8] hfiles 2024-11-24T04:22:44,740 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=58}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/682085848b1a4dc1a2b01cad48fbef3b for snapshot=snapshot_1732422164152_default_test-1732422103966 2024-11-24T04:22:44,740 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=58}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/4adb0dbc87f947bdaad87e7746c93da8 for snapshot=snapshot_1732422164152_default_test-1732422103966 2024-11-24T04:22:44,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741927_1103 (size=224) 2024-11-24T04:22:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T04:22:45,149 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=58}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:22:45,150 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=58}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=58 2024-11-24T04:22:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster(4169): Remote procedure done, pid=58 2024-11-24T04:22:45,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732422164152_default_test-1732422103966 on region 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:45,152 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:45,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-24T04:22:45,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 2c375e169f1afc2d9cce55928a15d594 in 573 msec 2024-11-24T04:22:45,160 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=snapshot_1732422164152_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T04:22:45,162 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=snapshot_1732422164152_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T04:22:45,164 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-24T04:22:45,165 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-24T04:22:45,165 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:22:45,166 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-24T04:22:45,166 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:22:45,167 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f2 2024-11-24T04:22:45,167 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:22:45,169 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/mobdir/data/default/test-1732422103966/3047fc58bb55744b4303d1cceb01bae9/mob/d41d8cd98f00b204e9800998ecf8427e20241124065620beac114f4a94d99cd6792a819b_2c375e169f1afc2d9cce55928a15d594] hfiles 2024-11-24T04:22:45,169 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/1): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/mobdir/data/default/test-1732422103966/3047fc58bb55744b4303d1cceb01bae9/mob/d41d8cd98f00b204e9800998ecf8427e20241124065620beac114f4a94d99cd6792a819b_2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:45,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741928_1104 (size=168) 2024-11-24T04:22:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T04:22:45,637 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=snapshot_1732422164152_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T04:22:45,637 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732422164152_default_test-1732422103966 2024-11-24T04:22:45,639 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_1732422164152_default_test-1732422103966 2024-11-24T04:22:45,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741929_1105 (size=1305) 2024-11-24T04:22:46,059 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=snapshot_1732422164152_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T04:22:46,071 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=snapshot_1732422164152_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T04:22:46,072 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_1732422164152_default_test-1732422103966 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_1732422164152_default_test-1732422103966 2024-11-24T04:22:46,074 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=snapshot_1732422164152_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T04:22:46,075 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732422164152_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 }, snapshot procedure id = 57 2024-11-24T04:22:46,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=snapshot_1732422164152_default_test-1732422103966 table=test-1732422103966 type=FLUSH ttl=0 } in 1.9090 sec 2024-11-24T04:22:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T04:22:46,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1732422103966 completed 2024-11-24T04:22:46,313 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1732422151095 2024-11-24T04:22:46,313 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-24T04:22:46,313 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 5 snapshots to be copied. 2024-11-24T04:22:46,313 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732422151095_backup:system_bulk 2024-11-24T04:22:46,313 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732422155493_backup_system_bulk to hdfs://localhost:45347/backupUT/backup_1732422151095/backup/system_bulk/ 2024-11-24T04:22:46,314 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-24T04:22:46,346 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:45347, inputRoot=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:22:46,346 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1702068436_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45347/backupUT/backup_1732422151095/backup/system_bulk, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45347/backupUT/backup_1732422151095/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1732422155493_backup_system_bulk 2024-11-24T04:22:46,348 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-24T04:22:46,352 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_1732422155493_backup_system_bulk to hdfs://localhost:45347/backupUT/backup_1732422151095/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1732422155493_backup_system_bulk 2024-11-24T04:22:46,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741930_1106 (size=76) 2024-11-24T04:22:46,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741931_1107 (size=702) 2024-11-24T04:22:46,766 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:46,766 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:46,767 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:47,638 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-16656104205041045792.jar 2024-11-24T04:22:47,638 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:47,638 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:47,638 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:47,638 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:47,639 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:47,639 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:47,639 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:47,639 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:47,639 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:47,640 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:22:47,640 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T04:22:47,640 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T04:22:47,640 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T04:22:47,640 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T04:22:47,641 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T04:22:47,641 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T04:22:47,641 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T04:22:47,641 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T04:22:47,641 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T04:22:47,642 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T04:22:47,642 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:47,642 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:47,642 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:47,642 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:47,642 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:47,643 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:47,643 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:47,676 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:22:47,699 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732422155493_backup_system_bulk' hfile list 2024-11-24T04:22:47,785 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local976461335_0005_9ff0fe90-b9c1-4231-9501-3e382ff3dd77/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-24T04:22:47,785 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local976461335_0005_9ff0fe90-b9c1-4231-9501-3e382ff3dd77/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-24T04:22:48,820 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-24T04:22:48,823 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-24T04:22:48,835 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732422155493_backup_system_bulk 2024-11-24T04:22:48,835 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732422155493_backup_system_bulk finished. 2024-11-24T04:22:48,835 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732422151095_ns2:test-17324221039661 2024-11-24T04:22:48,835 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732422157652_ns2_test-17324221039661 to hdfs://localhost:45347/backupUT/backup_1732422151095/ns2/test-17324221039661/ 2024-11-24T04:22:48,835 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-24T04:22:48,863 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:45347, inputRoot=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:22:48,863 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1702068436_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45347/backupUT/backup_1732422151095/ns2/test-17324221039661, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45347/backupUT/backup_1732422151095/ns2/test-17324221039661/.hbase-snapshot/.tmp/snapshot_1732422157652_ns2_test-17324221039661 2024-11-24T04:22:48,865 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-24T04:22:48,870 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_1732422157652_ns2_test-17324221039661 to hdfs://localhost:45347/backupUT/backup_1732422151095/ns2/test-17324221039661/.hbase-snapshot/.tmp/snapshot_1732422157652_ns2_test-17324221039661 2024-11-24T04:22:48,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741932_1108 (size=86) 2024-11-24T04:22:48,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741933_1109 (size=496) 2024-11-24T04:22:49,288 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:49,288 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:49,289 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:50,153 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-13689257389916161944.jar 2024-11-24T04:22:50,154 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:50,154 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:50,154 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:50,154 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:50,155 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:50,155 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:50,155 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:50,155 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:50,155 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:50,156 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:22:50,156 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T04:22:50,156 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T04:22:50,156 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T04:22:50,156 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T04:22:50,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T04:22:50,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T04:22:50,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T04:22:50,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T04:22:50,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T04:22:50,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T04:22:50,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:50,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:50,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:50,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:50,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:50,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:50,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:50,191 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:22:50,216 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732422157652_ns2_test-17324221039661' hfile list 2024-11-24T04:22:50,218 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=13.2 K 2024-11-24T04:22:50,288 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local1821392368_0006_58d60f32-c822-4f44-af64-3abdaef43798/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-24T04:22:50,288 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local1821392368_0006_58d60f32-c822-4f44-af64-3abdaef43798/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-24T04:22:50,359 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-24T04:22:50,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741934_1110 (size=8413) 2024-11-24T04:22:50,797 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/933ab35b68994dc8be3ccc706f0c0aa9 output=hdfs://localhost:45347/backupUT/backup_1732422151095/ns2/test-17324221039661/archive/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/933ab35b68994dc8be3ccc706f0c0aa9 2024-11-24T04:22:50,798 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-24T04:22:50,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741935_1111 (size=5134) 2024-11-24T04:22:51,227 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/1d220a07c36347908f109231fa9cf854 output=hdfs://localhost:45347/backupUT/backup_1732422151095/ns2/test-17324221039661/archive/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/1d220a07c36347908f109231fa9cf854 2024-11-24T04:22:51,227 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5134 (5.0 K) time=0sec 0.012M/sec 2024-11-24T04:22:51,327 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-24T04:22:51,328 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-24T04:22:51,339 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732422157652_ns2_test-17324221039661 2024-11-24T04:22:51,339 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732422157652_ns2_test-17324221039661 finished. 2024-11-24T04:22:51,339 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732422151095_ns4:test-17324221039663 2024-11-24T04:22:51,339 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732422159823_ns4_test-17324221039663 to hdfs://localhost:45347/backupUT/backup_1732422151095/ns4/test-17324221039663/ 2024-11-24T04:22:51,339 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-24T04:22:51,366 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:45347, inputRoot=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:22:51,366 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1702068436_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45347/backupUT/backup_1732422151095/ns4/test-17324221039663, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45347/backupUT/backup_1732422151095/ns4/test-17324221039663/.hbase-snapshot/.tmp/snapshot_1732422159823_ns4_test-17324221039663 2024-11-24T04:22:51,368 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-24T04:22:51,372 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_1732422159823_ns4_test-17324221039663 to hdfs://localhost:45347/backupUT/backup_1732422151095/ns4/test-17324221039663/.hbase-snapshot/.tmp/snapshot_1732422159823_ns4_test-17324221039663 2024-11-24T04:22:51,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741936_1112 (size=443) 2024-11-24T04:22:51,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741937_1113 (size=86) 2024-11-24T04:22:51,783 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:51,784 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:51,784 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:52,651 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-1392919705621029882.jar 2024-11-24T04:22:52,652 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:52,652 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:52,652 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:52,652 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:52,653 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:52,653 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:52,653 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:52,653 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:52,653 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:52,653 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:22:52,654 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T04:22:52,654 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T04:22:52,654 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T04:22:52,654 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T04:22:52,654 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T04:22:52,654 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T04:22:52,655 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T04:22:52,655 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T04:22:52,655 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T04:22:52,655 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T04:22:52,655 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:52,656 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:52,656 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:52,656 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:52,656 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:52,656 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:52,657 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:52,689 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:22:52,711 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732422159823_ns4_test-17324221039663' hfile list 2024-11-24T04:22:52,792 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local1711386734_0007_bfaf5351-dd54-451b-a0af-3ee39e43c709/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-24T04:22:52,792 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local1711386734_0007_bfaf5351-dd54-451b-a0af-3ee39e43c709/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-24T04:22:53,830 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-24T04:22:53,832 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-24T04:22:53,842 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732422159823_ns4_test-17324221039663 2024-11-24T04:22:53,842 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732422159823_ns4_test-17324221039663 finished. 2024-11-24T04:22:53,842 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732422151095_ns3:test-17324221039662 2024-11-24T04:22:53,842 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732422161983_ns3_test-17324221039662 to hdfs://localhost:45347/backupUT/backup_1732422151095/ns3/test-17324221039662/ 2024-11-24T04:22:53,842 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-24T04:22:53,869 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:45347, inputRoot=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:22:53,869 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1702068436_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45347/backupUT/backup_1732422151095/ns3/test-17324221039662, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45347/backupUT/backup_1732422151095/ns3/test-17324221039662/.hbase-snapshot/.tmp/snapshot_1732422161983_ns3_test-17324221039662 2024-11-24T04:22:53,871 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-24T04:22:53,875 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_1732422161983_ns3_test-17324221039662 to hdfs://localhost:45347/backupUT/backup_1732422151095/ns3/test-17324221039662/.hbase-snapshot/.tmp/snapshot_1732422161983_ns3_test-17324221039662 2024-11-24T04:22:53,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741939_1115 (size=443) 2024-11-24T04:22:53,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741938_1114 (size=86) 2024-11-24T04:22:54,290 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:54,290 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:54,291 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:55,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-15525988357869842283.jar 2024-11-24T04:22:55,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:55,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:55,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:55,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:55,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:55,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:55,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:55,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:55,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:55,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:22:55,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T04:22:55,162 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T04:22:55,162 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T04:22:55,162 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T04:22:55,162 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T04:22:55,162 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T04:22:55,162 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T04:22:55,163 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T04:22:55,163 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T04:22:55,163 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T04:22:55,163 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:55,163 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:55,164 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:55,164 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:55,164 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:55,164 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:55,164 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:55,198 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:22:55,222 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732422161983_ns3_test-17324221039662' hfile list 2024-11-24T04:22:55,301 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local1679318867_0008_71c050d6-1fa4-481f-adb9-cbcda62d14bf/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-24T04:22:55,301 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local1679318867_0008_71c050d6-1fa4-481f-adb9-cbcda62d14bf/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-24T04:22:56,341 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-24T04:22:56,343 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-24T04:22:56,351 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732422161983_ns3_test-17324221039662 2024-11-24T04:22:56,352 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732422161983_ns3_test-17324221039662 finished. 2024-11-24T04:22:56,352 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732422151095_test-1732422103966 2024-11-24T04:22:56,352 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732422164152_default_test-1732422103966 to hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/ 2024-11-24T04:22:56,352 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-24T04:22:56,377 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:45347, inputRoot=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:22:56,377 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1702068436_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/.hbase-snapshot/.tmp/snapshot_1732422164152_default_test-1732422103966 2024-11-24T04:22:56,379 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-24T04:22:56,385 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_1732422164152_default_test-1732422103966 to hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/.hbase-snapshot/.tmp/snapshot_1732422164152_default_test-1732422103966 2024-11-24T04:22:56,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741941_1117 (size=84) 2024-11-24T04:22:56,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741940_1116 (size=1305) 2024-11-24T04:22:56,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:56,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:56,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:57,671 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-1606674918557889967.jar 2024-11-24T04:22:57,671 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:57,671 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:57,672 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:57,672 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:57,672 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:57,672 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:57,672 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:57,672 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:57,673 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:22:57,673 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:22:57,673 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T04:22:57,673 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T04:22:57,673 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T04:22:57,673 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T04:22:57,674 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T04:22:57,674 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T04:22:57,674 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T04:22:57,674 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T04:22:57,674 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T04:22:57,674 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T04:22:57,675 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:57,675 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:57,675 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:57,675 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:57,675 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T04:22:57,675 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:57,676 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:22:57,707 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:22:57,735 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732422164152_default_test-1732422103966' hfile list 2024-11-24T04:22:57,738 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=43.0 K 2024-11-24T04:22:57,810 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local1661411316_0009_f72777ef-0efb-40f0-a198-ff6c9c049bc7/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-24T04:22:57,810 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local1661411316_0009_f72777ef-0efb-40f0-a198-ff6c9c049bc7/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-24T04:22:57,874 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-24T04:22:57,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741942_1118 (size=13760) 2024-11-24T04:22:58,307 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/4adb0dbc87f947bdaad87e7746c93da8 output=hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/4adb0dbc87f947bdaad87e7746c93da8 2024-11-24T04:22:58,308 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=13760 (13.4 K) time=0sec 0.031M/sec 2024-11-24T04:22:58,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741943_1119 (size=10796) 2024-11-24T04:22:58,737 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/60684cdd8d5e45cbaabfe6505beaad9f output=hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/60684cdd8d5e45cbaabfe6505beaad9f 2024-11-24T04:22:58,737 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=10796 (10.5 K) time=0sec 0.024M/sec 2024-11-24T04:22:58,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741944_1120 (size=8413) 2024-11-24T04:22:59,177 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3 output=hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3 2024-11-24T04:22:59,177 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-24T04:22:59,193 WARN [LocalJobRunner Map Task Executor #0 {}] mapred.Task(640): Task status: "copied 5.5 K/5.5 K (100.0%) from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/mobdir/data/default/test-1732422103966/3047fc58bb55744b4303d1cceb01bae9/mob/d41d8cd98f00b204e9800998ecf8427e20241124065620beac114f4a94d99cd6792a819b_2c375e169f1afc2d9cce55928a15d594 to hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/archive/data/default/test-1732422103966/3047fc58bb55744b4303d1cceb01bae9/mob/d41d8cd98f00b204e9800998ecf8427e20241124065620beac114f4a94d99cd6792a819b_2c375e169f1afc2d9cce55928a15d594" truncated to max limit (512 characters) 2024-11-24T04:22:59,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741945_1121 (size=5591) 2024-11-24T04:22:59,606 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/mobdir/data/default/test-1732422103966/3047fc58bb55744b4303d1cceb01bae9/mob/d41d8cd98f00b204e9800998ecf8427e20241124065620beac114f4a94d99cd6792a819b_2c375e169f1afc2d9cce55928a15d594 output=hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/archive/data/default/test-1732422103966/3047fc58bb55744b4303d1cceb01bae9/mob/d41d8cd98f00b204e9800998ecf8427e20241124065620beac114f4a94d99cd6792a819b_2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:22:59,606 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5591 (5.5 K) time=0sec 0.013M/sec 2024-11-24T04:22:59,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741946_1122 (size=5484) 2024-11-24T04:23:00,048 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/682085848b1a4dc1a2b01cad48fbef3b output=hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/682085848b1a4dc1a2b01cad48fbef3b 2024-11-24T04:23:00,048 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-11-24T04:23:00,853 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-24T04:23:00,856 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-24T04:23:00,865 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732422164152_default_test-1732422103966 2024-11-24T04:23:00,865 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732422164152_default_test-1732422103966 finished. 2024-11-24T04:23:00,865 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): backup:system_bulk 2024-11-24T04:23:00,865 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17324221039661 2024-11-24T04:23:00,865 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns4:test-17324221039663 2024-11-24T04:23:00,865 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns3:test-17324221039662 2024-11-24T04:23:00,865 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1732422103966 2024-11-24T04:23:00,876 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732422151095 2024-11-24T04:23:00,876 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-24T04:23:00,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741947_1123 (size=209) 2024-11-24T04:23:01,283 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45347/backupUT/backup_1732422151095/.backup.manifest 2024-11-24T04:23:01,283 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732422151095 finished: type=FULL,tablelist=backup:system_bulk;ns2:test-17324221039661;ns4:test-17324221039663;ns3:test-17324221039662;test-1732422103966,targetRootDir=hdfs://localhost:45347/backupUT,startts=1732422155292,completets=1732422180876,bytescopied=0 2024-11-24T04:23:01,283 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-24T04:23:01,283 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732422155493_backup_system_bulk 2024-11-24T04:23:01,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1732422155493_backup_system_bulk" type: DISABLED 2024-11-24T04:23:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732422155493_backup_system_bulk 2024-11-24T04:23:01,291 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732422155493_backup_system_bulk for backup backup_1732422151095 succeeded. 2024-11-24T04:23:01,292 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732422157652_ns2_test-17324221039661 2024-11-24T04:23:01,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1732422157652_ns2_test-17324221039661" type: DISABLED 2024-11-24T04:23:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732422157652_ns2_test-17324221039661 2024-11-24T04:23:01,298 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732422157652_ns2_test-17324221039661 for backup backup_1732422151095 succeeded. 2024-11-24T04:23:01,298 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732422159823_ns4_test-17324221039663 2024-11-24T04:23:01,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1732422159823_ns4_test-17324221039663" type: DISABLED 2024-11-24T04:23:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732422159823_ns4_test-17324221039663 2024-11-24T04:23:01,301 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732422159823_ns4_test-17324221039663 for backup backup_1732422151095 succeeded. 2024-11-24T04:23:01,301 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732422161983_ns3_test-17324221039662 2024-11-24T04:23:01,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1732422161983_ns3_test-17324221039662" type: DISABLED 2024-11-24T04:23:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732422161983_ns3_test-17324221039662 2024-11-24T04:23:01,305 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732422161983_ns3_test-17324221039662 for backup backup_1732422151095 succeeded. 2024-11-24T04:23:01,305 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732422164152_default_test-1732422103966 2024-11-24T04:23:01,305 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1732422164152_default_test-1732422103966" type: DISABLED 2024-11-24T04:23:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732422164152_default_test-1732422103966 2024-11-24T04:23:01,308 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732422164152_default_test-1732422103966 for backup backup_1732422151095 succeeded. 2024-11-24T04:23:01,308 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-24T04:23:01,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-11-24T04:23:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-24T04:23:01,315 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-24T04:23:01,317 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-24T04:23:01,323 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732422151095 completed. 2024-11-24T04:23:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:02,380 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T04:23:06,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:06,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:06,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:06,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:06,468 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-24T04:23:06,472 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732422186353 2024-11-24T04:23:06,474 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732422151095} 2024-11-24T04:23:06,474 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-24T04:23:06,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:06,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:06,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:06,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:06,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:06,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:06,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:06,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:06,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:06,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:06,574 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T04:23:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732422186574 (current time:1732422186574). 2024-11-24T04:23:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T04:23:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-24T04:23:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a489b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:06,576 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:06,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:06,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:06,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67e1aa3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:06,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:06,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:06,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:06,578 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:06,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bece2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:06,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:06,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:06,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:06,582 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719. 2024-11-24T04:23:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:06,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T04:23:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T04:23:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 59 2024-11-24T04:23:06,585 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T04:23:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T04:23:06,586 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T04:23:06,588 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T04:23:06,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741948_1124 (size=52) 2024-11-24T04:23:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T04:23:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T04:23:06,998 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T04:23:06,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2}] 2024-11-24T04:23:06,999 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:23:07,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36347 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=60 2024-11-24T04:23:07,151 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:23:07,152 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] regionserver.HRegion(2902): Flushing 5a030641752b7def7c38c5f6295501b2 2/2 column families, dataSize=2.54 KB heapSize=4.22 KB 2024-11-24T04:23:07,157 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/meta/1dfb9c8318474ca2873178116fcf6a13 is 158, key is trslm:hdfs://localhost:45347/backupUT\x00ns2:test-17324221039661/meta:log-roll-map/1732422180869/Put/seqid=0 2024-11-24T04:23:07,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741949_1125 (size=6646) 2024-11-24T04:23:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T04:23:07,561 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/meta/1dfb9c8318474ca2873178116fcf6a13 2024-11-24T04:23:07,571 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/session/4d874203eaa94868bfb9d2d6e5168989 is 826, key is session:backup_1732422151095/session:context/1732422181316/Put/seqid=0 2024-11-24T04:23:07,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741950_1126 (size=5928) 2024-11-24T04:23:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T04:23:07,977 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/session/4d874203eaa94868bfb9d2d6e5168989 2024-11-24T04:23:07,985 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/meta/1dfb9c8318474ca2873178116fcf6a13 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/1dfb9c8318474ca2873178116fcf6a13 2024-11-24T04:23:07,993 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/1dfb9c8318474ca2873178116fcf6a13, entries=12, sequenceid=42, filesize=6.5 K 2024-11-24T04:23:07,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/session/4d874203eaa94868bfb9d2d6e5168989 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/4d874203eaa94868bfb9d2d6e5168989 2024-11-24T04:23:08,001 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/4d874203eaa94868bfb9d2d6e5168989, entries=2, sequenceid=42, filesize=5.8 K 2024-11-24T04:23:08,002 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] regionserver.HRegion(3140): Finished flush of dataSize ~2.54 KB/2604, heapSize ~4.19 KB/4288, currentSize=0 B/0 for 5a030641752b7def7c38c5f6295501b2 in 851ms, sequenceid=42, compaction requested=true 2024-11-24T04:23:08,002 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] regionserver.HRegion(2603): Flush status journal for 5a030641752b7def7c38c5f6295501b2: 2024-11-24T04:23:08,002 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. for snapshot_backup_system completed. 2024-11-24T04:23:08,002 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2.' region-info for snapshot=snapshot_backup_system 2024-11-24T04:23:08,002 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T04:23:08,002 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/d61b2831ca274cc7989a43ee1404fb09, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/09622658e56b4cb896386e23d2791852, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/1dfb9c8318474ca2873178116fcf6a13] hfiles 2024-11-24T04:23:08,002 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/d61b2831ca274cc7989a43ee1404fb09 for snapshot=snapshot_backup_system 2024-11-24T04:23:08,002 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/09622658e56b4cb896386e23d2791852 for snapshot=snapshot_backup_system 2024-11-24T04:23:08,002 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/1dfb9c8318474ca2873178116fcf6a13 for snapshot=snapshot_backup_system 2024-11-24T04:23:08,002 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/e44333cb30954234bdc3a05f162b96cd, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/fea54c00fd9144c2b57bfc7379644a05, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/c3f9856af53640e1aa92faceb2b4a6bd, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/4d874203eaa94868bfb9d2d6e5168989] hfiles 2024-11-24T04:23:08,002 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] snapshot.SnapshotManifest(265): Adding reference for file (1/4): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/e44333cb30954234bdc3a05f162b96cd for snapshot=snapshot_backup_system 2024-11-24T04:23:08,002 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] snapshot.SnapshotManifest(265): Adding reference for file (2/4): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/fea54c00fd9144c2b57bfc7379644a05 for snapshot=snapshot_backup_system 2024-11-24T04:23:08,003 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] snapshot.SnapshotManifest(265): Adding reference for file (3/4): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/c3f9856af53640e1aa92faceb2b4a6bd for snapshot=snapshot_backup_system 2024-11-24T04:23:08,003 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] snapshot.SnapshotManifest(265): Adding reference for file (4/4): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/4d874203eaa94868bfb9d2d6e5168989 for snapshot=snapshot_backup_system 2024-11-24T04:23:08,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741951_1127 (size=330) 2024-11-24T04:23:08,410 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:23:08,410 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60 2024-11-24T04:23:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster(4169): Remote procedure done, pid=60 2024-11-24T04:23:08,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:23:08,413 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:23:08,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-24T04:23:08,420 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T04:23:08,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 5a030641752b7def7c38c5f6295501b2 in 1.4170 sec 2024-11-24T04:23:08,422 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T04:23:08,423 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T04:23:08,423 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-24T04:23:08,424 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-24T04:23:08,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741952_1128 (size=967) 2024-11-24T04:23:08,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T04:23:08,840 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T04:23:08,851 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T04:23:08,851 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.hbase-snapshot/snapshot_backup_system 2024-11-24T04:23:08,853 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T04:23:08,853 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 59 2024-11-24T04:23:08,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2700 sec 2024-11-24T04:23:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T04:23:10,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-24T04:23:10,753 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732422186353 started at 1732422190753. 2024-11-24T04:23:10,757 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732422186353 has been started. 2024-11-24T04:23:10,763 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [backup:system_bulk, ns2:test-17324221039661, ns3:test-17324221039662, ns4:test-17324221039663, test-1732422103966] 2024-11-24T04:23:10,768 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1732422141631for backupID backup_1732422186353 2024-11-24T04:23:10,768 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-24T04:23:10,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-11-24T04:23:10,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-24T04:23:10,769 INFO [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-24T04:23:10,769 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-24T04:23:10,769 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-24T04:23:10,769 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-24T04:23:10,770 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T04:23:10,770 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T04:23:10,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T04:23:10,842 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,843 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T04:23:10,843 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T04:23:10,844 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,844 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-24T04:23:10,845 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-24T04:23:10,846 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T04:23:10,848 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-24T04:23:10,848 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T04:23:10,848 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-24T04:23:10,849 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-24T04:23:10,849 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-24T04:23:10,849 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-24T04:23:10,849 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-24T04:23:10,849 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-24T04:23:10,849 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-24T04:23:10,850 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'e57b7116ff5e,36347,1732422095008' joining acquired barrier for procedure (rolllog) in zk 2024-11-24T04:23:10,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,866 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-24T04:23:10,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T04:23:10,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T04:23:10,867 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-24T04:23:10,868 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-24T04:23:10,868 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T04:23:10,869 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T04:23:10,870 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:23:10,871 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,872 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T04:23:10,873 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'e57b7116ff5e,36347,1732422095008' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-24T04:23:10,873 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@70fbf935[Count = 0] remaining members to acquire global barrier 2024-11-24T04:23:10,873 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-24T04:23:10,874 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-24T04:23:10,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T04:23:10,887 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-24T04:23:10,887 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-24T04:23:10,887 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-24T04:23:10,887 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,887 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-24T04:23:10,887 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-24T04:23:10,888 DEBUG [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,888 INFO [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732422155364 highest: 1732422155400 on e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,888 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0:(num 1732422155364) roll requested 2024-11-24T04:23:10,905 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422190889, exclude list is [], retry=0 2024-11-24T04:23:10,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:23:10,909 INFO [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422155364 with entries=2, filesize=645 B; new WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422190889 2024-11-24T04:23:10,909 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:23:10,909 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422155364 is not closed yet, will try archiving it next time 2024-11-24T04:23:10,909 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL e57b7116ff5e%2C36347%2C1732422095008.meta:.meta(num 1732422155383) roll requested 2024-11-24T04:23:10,909 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422122082 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422122082 2024-11-24T04:23:10,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741910_1086 (size=653) 2024-11-24T04:23:10,911 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422155364 not finished, retry = 0 2024-11-24T04:23:10,921 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422190909.meta, exclude list is [], retry=0 2024-11-24T04:23:10,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:23:10,925 INFO [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422155383.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422190909.meta 2024-11-24T04:23:10,925 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:23:10,925 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422155383.meta is not closed yet, will try archiving it next time 2024-11-24T04:23:10,925 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1:(num 1732422155400) roll requested 2024-11-24T04:23:10,926 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422155383.meta not finished, retry = 0 2024-11-24T04:23:10,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741911_1087 (size=93) 2024-11-24T04:23:10,937 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422190925, exclude list is [], retry=0 2024-11-24T04:23:10,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44025,DS-9c46ac66-0d52-4f6c-b4c1-05c453fbf35c,DISK] 2024-11-24T04:23:10,941 INFO [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422155400 with entries=17, filesize=4.59 KB; new WAL /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422190925 2024-11-24T04:23:10,941 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40087:40087)] 2024-11-24T04:23:10,941 DEBUG [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422155400 is not closed yet, will try archiving it next time 2024-11-24T04:23:10,941 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422141631 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422141631 2024-11-24T04:23:10,943 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422155400 not finished, retry = 0 2024-11-24T04:23:10,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741912_1088 (size=4710) 2024-11-24T04:23:10,949 DEBUG [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-11-24T04:23:10,949 INFO [rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732422190889 on e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:10,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:10,967 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-24T04:23:10,967 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'e57b7116ff5e,36347,1732422095008' in zk 2024-11-24T04:23:10,981 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-24T04:23:10,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,981 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T04:23:10,981 DEBUG [member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-24T04:23:10,981 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,981 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T04:23:10,981 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T04:23:10,981 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T04:23:10,982 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T04:23:10,982 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:23:10,983 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,983 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T04:23:10,983 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:23:10,984 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,984 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'e57b7116ff5e,36347,1732422095008': 2024-11-24T04:23:10,985 INFO [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-24T04:23:10,985 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'e57b7116ff5e,36347,1732422095008' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-24T04:23:10,985 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-24T04:23:10,985 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-24T04:23:10,985 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-24T04:23:10,985 INFO [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-24T04:23:10,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T04:23:10,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T04:23:10,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T04:23:10,993 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-24T04:23:10,993 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T04:23:10,993 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-24T04:23:10,993 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T04:23:10,993 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T04:23:10,993 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T04:23:10,993 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T04:23:10,994 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T04:23:10,994 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,994 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:23:10,994 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T04:23:10,995 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T04:23:10,995 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:23:10,996 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,996 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:10,996 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T04:23:10,997 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T04:23:10,997 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:11,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:11,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T04:23:11,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T04:23:11,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T04:23:11,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:11,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T04:23:11,003 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T04:23:11,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T04:23:11,003 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T04:23:11,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T04:23:11,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T04:23:11,004 DEBUG [(e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T04:23:11,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-24T04:23:11,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-24T04:23:11,004 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2024-11-24T04:23:11,004 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T04:23:11,004 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T04:23:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-24T04:23:11,012 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422155364 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422155364 2024-11-24T04:23:11,015 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {e57b7116ff5e:36347=1732422141631} newestTimestamps: {e57b7116ff5e:36347=1732422155400} 2024-11-24T04:23:11,017 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: e57b7116ff5e%2C36347%2C1732422095008.meta.1732422098687.meta 2024-11-24T04:23:11,017 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: e57b7116ff5e%2C36347%2C1732422095008.meta.1732422141614.meta 2024-11-24T04:23:11,017 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: e57b7116ff5e%2C36347%2C1732422095008.meta.1732422155383.meta 2024-11-24T04:23:11,017 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: e57b7116ff5e%2C36347%2C1732422095008.meta.1732422190909.meta 2024-11-24T04:23:11,017 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422190889 2024-11-24T04:23:11,018 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422155400 2024-11-24T04:23:11,018 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422190925 2024-11-24T04:23:11,020 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422122100.meta 2024-11-24T04:23:11,027 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422155383.meta to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs/e57b7116ff5e%2C36347%2C1732422095008.meta.1732422155383.meta 2024-11-24T04:23:11,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741956_1132 (size=640) 2024-11-24T04:23:11,437 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:backup:system_bulk target: hdfs://localhost:45347/backupUT/backup_1732422186353/backup/system_bulk descriptor: 'backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:23:11,437 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-24T04:23:11,442 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table backup:system_bulk 2024-11-24T04:23:11,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741957_1133 (size=45) 2024-11-24T04:23:11,850 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table backup:system_bulk 2024-11-24T04:23:11,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741958_1134 (size=364) 2024-11-24T04:23:12,270 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17324221039661 target: hdfs://localhost:45347/backupUT/backup_1732422186353/ns2/test-17324221039661 descriptor: 'ns2:test-17324221039661', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:23:12,270 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-24T04:23:12,274 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17324221039661 2024-11-24T04:23:12,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741959_1135 (size=50) 2024-11-24T04:23:12,684 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17324221039661 2024-11-24T04:23:12,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741960_1136 (size=390) 2024-11-24T04:23:13,105 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns4:test-17324221039663 target: hdfs://localhost:45347/backupUT/backup_1732422186353/ns4/test-17324221039663 descriptor: 'ns4:test-17324221039663', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:23:13,105 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-24T04:23:13,112 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns4:test-17324221039663 2024-11-24T04:23:13,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741961_1137 (size=50) 2024-11-24T04:23:13,519 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns4:test-17324221039663 2024-11-24T04:23:13,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741962_1138 (size=390) 2024-11-24T04:23:13,938 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns3:test-17324221039662 target: hdfs://localhost:45347/backupUT/backup_1732422186353/ns3/test-17324221039662 descriptor: 'ns3:test-17324221039662', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:23:13,938 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-24T04:23:13,942 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns3:test-17324221039662 2024-11-24T04:23:13,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741963_1139 (size=50) 2024-11-24T04:23:14,351 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns3:test-17324221039662 2024-11-24T04:23:14,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741964_1140 (size=908) 2024-11-24T04:23:14,768 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1732422103966 target: hdfs://localhost:45347/backupUT/backup_1732422186353/default/test-1732422103966 descriptor: 'test-1732422103966', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:23:14,768 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-24T04:23:14,773 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1732422103966 2024-11-24T04:23:14,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741965_1141 (size=53) 2024-11-24T04:23:15,181 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1732422103966 2024-11-24T04:23:15,198 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:45347/backupUT/.tmp/backup_1732422186353 from hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422155400;hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422155364 2024-11-24T04:23:15,199 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ac65a93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:15,199 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:15,199 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:15,200 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:15,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:15,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:15,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@163af48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:15,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:15,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:15,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:15,202 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:15,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@390c0e24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:15,203 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:15,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:15,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3112c86a 2024-11-24T04:23:15,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:15,204 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:15,208 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-24T04:23:15,208 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table backup:system_bulk 2024-11-24T04:23:15,208 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:15,208 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:15,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:15,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:15,210 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:15,211 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for backup:system_bulk: backup:system_bulk; 2024-11-24T04:23:15,211 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17324221039661 2024-11-24T04:23:15,211 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17324221039661,, stopping at row=ns2:test-17324221039661 ,, for max=2147483647 with caching=100 2024-11-24T04:23:15,213 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17324221039661: ns2:test-17324221039661; 2024-11-24T04:23:15,213 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns3:test-17324221039662 2024-11-24T04:23:15,213 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns3:test-17324221039662,, stopping at row=ns3:test-17324221039662 ,, for max=2147483647 with caching=100 2024-11-24T04:23:15,215 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns3:test-17324221039662: ns3:test-17324221039662; 2024-11-24T04:23:15,215 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns4:test-17324221039663 2024-11-24T04:23:15,215 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns4:test-17324221039663,, stopping at row=ns4:test-17324221039663 ,, for max=2147483647 with caching=100 2024-11-24T04:23:15,216 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns4:test-17324221039663: ns4:test-17324221039663; 2024-11-24T04:23:15,216 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1732422103966 2024-11-24T04:23:15,216 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732422103966,, stopping at row=test-1732422103966 ,, for max=2147483647 with caching=100 2024-11-24T04:23:15,219 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1732422103966: test-1732422103966; 2024-11-24T04:23:15,219 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 5 reduce partitions to match current region count for all tables 2024-11-24T04:23:15,219 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/partitions_9ceb5769-bee2-4078-af7b-366bcddb6e70 2024-11-24T04:23:15,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741966_1142 (size=324) 2024-11-24T04:23:15,629 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:15,630 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:15,631 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:15,928 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 2c375e169f1afc2d9cce55928a15d594, had cached 0 bytes from a total of 38453 2024-11-24T04:23:16,495 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-143536664644398648.jar 2024-11-24T04:23:16,495 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:16,496 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:16,496 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:16,496 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:16,496 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:16,497 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:16,497 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:16,497 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:16,497 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:16,498 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:23:16,498 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T04:23:16,498 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T04:23:16,499 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T04:23:16,499 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T04:23:16,499 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T04:23:16,499 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T04:23:16,500 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T04:23:16,500 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T04:23:16,500 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T04:23:16,500 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T04:23:16,501 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:16,501 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:16,501 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:16,501 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:16,502 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:16,502 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:16,502 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:23:16,502 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: backup:system_bulk,ns2:test-17324221039661,ns3:test-17324221039662,ns4:test-17324221039663,default:test-1732422103966 2024-11-24T04:23:16,502 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:23:16,503 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:16,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:16,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:16,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:16,503 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:23:17,367 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-14109434452080736542.jar 2024-11-24T04:23:17,377 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:23:17,403 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422155400 for WAL files 2024-11-24T04:23:17,404 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422155400 2024-11-24T04:23:17,405 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422155364 for WAL files 2024-11-24T04:23:17,405 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422155364 2024-11-24T04:23:17,483 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local1557208203_0010_2c0cf275-f6d5-4507-a1d5-c49b3635d52c/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-24T04:23:17,483 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local1557208203_0010_2c0cf275-f6d5-4507-a1d5-c49b3635d52c/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-24T04:23:17,526 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422155400 for hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422155400 (-9223372036854775808:9223372036854775807) length:4710 2024-11-24T04:23:17,530 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-24T04:23:17,530 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-24T04:23:17,562 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422155364 for hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422155364 (-9223372036854775808:9223372036854775807) length:653 2024-11-24T04:23:17,564 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-24T04:23:17,565 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-24T04:23:17,636 DEBUG [pool-260-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528aa645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:17,636 DEBUG [pool-260-thread-1 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:17,637 DEBUG [pool-260-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:17,637 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:17,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:17,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:17,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e64cd74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:17,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:17,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:17,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:17,639 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:17,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d59771, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:17,640 DEBUG [pool-260-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:17,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:17,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:17,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:17,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732422103966', row='row-test-1732422103966-2-0', locateType=CURRENT is [region=test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594., hostname=e57b7116ff5e,36347,1732422095008, seqNum=338] 2024-11-24T04:23:17,644 INFO [pool-260-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-260-thread-1. 2024-11-24T04:23:17,644 DEBUG [pool-260-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:17,644 DEBUG [pool-260-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:17,644 DEBUG [pool-260-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:17,644 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:17,644 DEBUG [pool-260-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732422103966-2-0] 2024-11-24T04:23:17,644 DEBUG [pool-260-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: e57b7116ff5e 2024-11-24T04:23:17,647 DEBUG [pool-260-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/backupUT/.tmp/backup_1732422186353/_temporary/0/_temporary/attempt_local1557208203_0010_r_000004_0/default/test-1732422103966/f2/0f50a97b767c46f78559b14c3a21db99 is 58, key is row-test-1732422103966-2-0/f2:q1/1732422181343/Put/seqid=0 2024-11-24T04:23:17,649 WARN [Thread-1513 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [e57b7116ff5e:36347] Specified favored nodes: [e57b7116ff5e:36347] 2024-11-24T04:23:17,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741967_1143 (size=5367) 2024-11-24T04:23:18,028 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region d3f34500755154053262a8d9790ac0a1, had cached 0 bytes from a total of 13547 2024-11-24T04:23:18,523 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:45347/backupUT 2024-11-24T04:23:18,523 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1732422186353 2024-11-24T04:23:18,523 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-11-24T04:23:18,534 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:45347/backupUT/.tmp/backup_1732422186353, hdfs://localhost:45347/backupUT] 2024-11-24T04:23:18,580 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:23:18,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741968_1144 (size=5367) 2024-11-24T04:23:19,187 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-11-24T04:23:19,190 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1732422186353 2024-11-24T04:23:19,190 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 5367 bytes copied." 2024-11-24T04:23:19,191 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local913379523_0011 completed: true true 2024-11-24T04:23:19,191 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=6668609 FILE: Number of bytes written=17205408 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=409247 HDFS: Number of bytes written=2547251 HDFS: Number of read operations=1162 HDFS: Number of large read operations=0 HDFS: Number of write operations=421 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=1482 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=5367 Bytes Copied=5367 Bytes Expected=5367 Files Copied=2 DIR_COPY=4 2024-11-24T04:23:19,192 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:45347/backupUT/.tmp/backup_1732422186353 to hdfs://localhost:45347/backupUT finished. 2024-11-24T04:23:19,215 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732422186353 2024-11-24T04:23:19,219 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732422151095} 2024-11-24T04:23:19,219 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-24T04:23:19,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741969_1145 (size=714) 2024-11-24T04:23:19,627 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45347/backupUT/backup_1732422186353/.backup.manifest 2024-11-24T04:23:19,628 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732422186353 finished: type=INCREMENTAL,tablelist=backup:system_bulk;ns2:test-17324221039661;ns4:test-17324221039663;ns3:test-17324221039662;test-1732422103966,targetRootDir=hdfs://localhost:45347/backupUT,startts=1732422190753,completets=1732422199215,bytescopied=0 2024-11-24T04:23:19,628 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-24T04:23:19,639 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-11-24T04:23:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-24T04:23:19,643 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-24T04:23:19,646 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-24T04:23:19,649 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732422186353 completed. 2024-11-24T04:23:19,649 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-11-24T04:23:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:19,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:19,664 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(191): Restoring full backup_1732422115214 2024-11-24T04:23:19,667 INFO [Time-limited test {}] impl.RestoreTablesClient(100): HBase table table1 does not exist. It will be created during restore process 2024-11-24T04:23:19,668 INFO [Time-limited test {}] impl.RestoreTablesClient(100): HBase table ns2:table2 does not exist. It will be created during restore process 2024-11-24T04:23:19,668 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45347/backupUT/backup_1732422115214/.backup.manifest 2024-11-24T04:23:19,670 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732422115214/.backup.manifest 2024-11-24T04:23:19,673 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45347/backupUT/backup_1732422115214/.backup.manifest 2024-11-24T04:23:19,675 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732422115214/.backup.manifest 2024-11-24T04:23:19,675 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'test-1732422103966' to 'table1' from full backup image hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966 2024-11-24T04:23:19,683 INFO [Time-limited test {}] util.RestoreTool(491): Creating target table 'table1' 2024-11-24T04:23:19,683 DEBUG [Time-limited test {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:23:19,684 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f in region [hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594] 2024-11-24T04:23:19,688 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3 first=row0 last=row98 2024-11-24T04:23:19,688 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f3 in region [hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594] 2024-11-24T04:23:19,692 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f3/f9440cbd308f4fa3b08696b88b8a8b86 first=row-test-1732422103966-3-0 last=row-test-1732422103966-3-5 2024-11-24T04:23:19,692 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob in region [hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594] 2024-11-24T04:23:19,695 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/682085848b1a4dc1a2b01cad48fbef3b first=row-test-1732422103966-3-0 last=row-test-1732422103966-3-5 2024-11-24T04:23:19,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'table1', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:23:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-24T04:23:19,701 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T04:23:19,701 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:19,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 61 2024-11-24T04:23:19,702 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T04:23:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T04:23:19,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741970_1146 (size=938) 2024-11-24T04:23:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T04:23:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T04:23:20,112 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a039ab1cc3e38eda24eec1b755eec11e, NAME => 'table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e.', STARTKEY => '', ENDKEY => 'row0'}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:23:20,113 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 949a2f97f8cb299ab38ebab18f920edc, NAME => 'table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc.', STARTKEY => 'row0', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:23:20,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741972_1148 (size=45) 2024-11-24T04:23:20,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741971_1147 (size=45) 2024-11-24T04:23:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T04:23:20,529 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:23:20,529 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(898): Instantiated table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:23:20,529 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing a039ab1cc3e38eda24eec1b755eec11e, disabling compactions & flushes 2024-11-24T04:23:20,529 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1722): Closing 949a2f97f8cb299ab38ebab18f920edc, disabling compactions & flushes 2024-11-24T04:23:20,530 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1755): Closing region table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. 2024-11-24T04:23:20,530 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. 2024-11-24T04:23:20,530 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. 2024-11-24T04:23:20,530 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. 2024-11-24T04:23:20,530 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. after waiting 0 ms 2024-11-24T04:23:20,530 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. after waiting 0 ms 2024-11-24T04:23:20,530 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. 2024-11-24T04:23:20,530 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. 2024-11-24T04:23:20,530 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1973): Closed table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. 2024-11-24T04:23:20,530 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. 2024-11-24T04:23:20,531 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for a039ab1cc3e38eda24eec1b755eec11e: Waiting for close lock at 1732422200529Disabling compacts and flushes for region at 1732422200529Disabling writes for close at 1732422200530 (+1 ms)Writing region close event to WAL at 1732422200530Closed at 1732422200530 2024-11-24T04:23:20,531 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1676): Region close journal for 949a2f97f8cb299ab38ebab18f920edc: Waiting for close lock at 1732422200529Disabling compacts and flushes for region at 1732422200529Disabling writes for close at 1732422200530 (+1 ms)Writing region close event to WAL at 1732422200530Closed at 1732422200530 2024-11-24T04:23:20,533 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T04:23:20,533 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732422200533"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422200533"}]},"ts":"1732422200533"} 2024-11-24T04:23:20,533 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732422200533"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422200533"}]},"ts":"1732422200533"} 2024-11-24T04:23:20,538 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 2 regions to meta. 2024-11-24T04:23:20,539 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T04:23:20,539 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422200539"}]},"ts":"1732422200539"} 2024-11-24T04:23:20,541 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-24T04:23:20,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=a039ab1cc3e38eda24eec1b755eec11e, ASSIGN}, {pid=63, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=949a2f97f8cb299ab38ebab18f920edc, ASSIGN}] 2024-11-24T04:23:20,543 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=949a2f97f8cb299ab38ebab18f920edc, ASSIGN 2024-11-24T04:23:20,543 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=a039ab1cc3e38eda24eec1b755eec11e, ASSIGN 2024-11-24T04:23:20,544 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=a039ab1cc3e38eda24eec1b755eec11e, ASSIGN; state=OFFLINE, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=false 2024-11-24T04:23:20,544 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=949a2f97f8cb299ab38ebab18f920edc, ASSIGN; state=OFFLINE, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=false 2024-11-24T04:23:20,696 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=949a2f97f8cb299ab38ebab18f920edc, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:20,696 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=a039ab1cc3e38eda24eec1b755eec11e, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:20,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=a039ab1cc3e38eda24eec1b755eec11e, ASSIGN because future has completed 2024-11-24T04:23:20,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure a039ab1cc3e38eda24eec1b755eec11e, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:23:20,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=949a2f97f8cb299ab38ebab18f920edc, ASSIGN because future has completed 2024-11-24T04:23:20,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 949a2f97f8cb299ab38ebab18f920edc, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:23:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T04:23:20,864 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] handler.AssignRegionHandler(132): Open table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. 2024-11-24T04:23:20,865 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => a039ab1cc3e38eda24eec1b755eec11e, NAME => 'table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e.', STARTKEY => '', ENDKEY => 'row0'} 2024-11-24T04:23:20,866 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:23:20,866 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:20,866 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(898): Instantiated table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:23:20,867 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(7794): checking encryption for a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:20,867 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(7797): checking classloading for a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:20,870 INFO [StoreOpener-a039ab1cc3e38eda24eec1b755eec11e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:20,872 INFO [StoreOpener-a039ab1cc3e38eda24eec1b755eec11e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a039ab1cc3e38eda24eec1b755eec11e columnFamilyName f 2024-11-24T04:23:20,872 DEBUG [StoreOpener-a039ab1cc3e38eda24eec1b755eec11e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:20,873 INFO [StoreOpener-a039ab1cc3e38eda24eec1b755eec11e-1 {}] regionserver.HStore(327): Store=a039ab1cc3e38eda24eec1b755eec11e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:20,873 INFO [StoreOpener-a039ab1cc3e38eda24eec1b755eec11e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:20,874 INFO [StoreOpener-a039ab1cc3e38eda24eec1b755eec11e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a039ab1cc3e38eda24eec1b755eec11e columnFamilyName f3 2024-11-24T04:23:20,874 DEBUG [StoreOpener-a039ab1cc3e38eda24eec1b755eec11e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:20,875 INFO [StoreOpener-a039ab1cc3e38eda24eec1b755eec11e-1 {}] regionserver.HStore(327): Store=a039ab1cc3e38eda24eec1b755eec11e/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:20,875 INFO [StoreOpener-a039ab1cc3e38eda24eec1b755eec11e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:20,876 INFO [StoreOpener-a039ab1cc3e38eda24eec1b755eec11e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a039ab1cc3e38eda24eec1b755eec11e columnFamilyName mob 2024-11-24T04:23:20,877 DEBUG [StoreOpener-a039ab1cc3e38eda24eec1b755eec11e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:20,878 INFO [StoreOpener-a039ab1cc3e38eda24eec1b755eec11e-1 {}] regionserver.HStore(327): Store=a039ab1cc3e38eda24eec1b755eec11e/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:20,878 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(1038): replaying wal for a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:20,879 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:20,879 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:20,881 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(1048): stopping wal replay for a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:20,881 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:20,882 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:23:20,883 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(1093): writing seq id for a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:20,886 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:23:20,887 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(1114): Opened a039ab1cc3e38eda24eec1b755eec11e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11029648320, jitterRate=0.027216047048568726}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:23:20,887 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:20,888 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(1006): Region open journal for a039ab1cc3e38eda24eec1b755eec11e: Running coprocessor pre-open hook at 1732422200867Writing region info on filesystem at 1732422200867Initializing all the Stores at 1732422200870 (+3 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422200870Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422200870Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422200870Cleaning up temporary data from old regions at 1732422200881 (+11 ms)Running coprocessor post-open hooks at 1732422200887 (+6 ms)Region opened successfully at 1732422200888 (+1 ms) 2024-11-24T04:23:20,889 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e., pid=64, masterSystemTime=1732422200856 2024-11-24T04:23:20,892 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. 2024-11-24T04:23:20,892 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] handler.AssignRegionHandler(153): Opened table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. 2024-11-24T04:23:20,892 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. 2024-11-24T04:23:20,892 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 949a2f97f8cb299ab38ebab18f920edc, NAME => 'table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc.', STARTKEY => 'row0', ENDKEY => ''} 2024-11-24T04:23:20,892 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:23:20,892 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=a039ab1cc3e38eda24eec1b755eec11e, regionState=OPEN, openSeqNum=2, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:20,893 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:20,893 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:23:20,893 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:20,893 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:20,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure a039ab1cc3e38eda24eec1b755eec11e, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:23:20,895 INFO [StoreOpener-949a2f97f8cb299ab38ebab18f920edc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:20,896 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46719 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=e57b7116ff5e,36347,1732422095008, table=table1, region=a039ab1cc3e38eda24eec1b755eec11e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T04:23:20,897 INFO [StoreOpener-949a2f97f8cb299ab38ebab18f920edc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 949a2f97f8cb299ab38ebab18f920edc columnFamilyName f 2024-11-24T04:23:20,897 DEBUG [StoreOpener-949a2f97f8cb299ab38ebab18f920edc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:20,898 INFO [StoreOpener-949a2f97f8cb299ab38ebab18f920edc-1 {}] regionserver.HStore(327): Store=949a2f97f8cb299ab38ebab18f920edc/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:20,898 INFO [StoreOpener-949a2f97f8cb299ab38ebab18f920edc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:20,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=62 2024-11-24T04:23:20,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure a039ab1cc3e38eda24eec1b755eec11e, server=e57b7116ff5e,36347,1732422095008 in 194 msec 2024-11-24T04:23:20,900 INFO [StoreOpener-949a2f97f8cb299ab38ebab18f920edc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 949a2f97f8cb299ab38ebab18f920edc columnFamilyName f3 2024-11-24T04:23:20,900 DEBUG [StoreOpener-949a2f97f8cb299ab38ebab18f920edc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:20,901 INFO [StoreOpener-949a2f97f8cb299ab38ebab18f920edc-1 {}] regionserver.HStore(327): Store=949a2f97f8cb299ab38ebab18f920edc/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:20,901 INFO [StoreOpener-949a2f97f8cb299ab38ebab18f920edc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:20,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=a039ab1cc3e38eda24eec1b755eec11e, ASSIGN in 357 msec 2024-11-24T04:23:20,902 INFO [StoreOpener-949a2f97f8cb299ab38ebab18f920edc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 949a2f97f8cb299ab38ebab18f920edc columnFamilyName mob 2024-11-24T04:23:20,903 DEBUG [StoreOpener-949a2f97f8cb299ab38ebab18f920edc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:20,903 INFO [StoreOpener-949a2f97f8cb299ab38ebab18f920edc-1 {}] regionserver.HStore(327): Store=949a2f97f8cb299ab38ebab18f920edc/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:20,904 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:20,904 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:20,905 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:20,906 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:20,906 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:20,906 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:23:20,908 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:20,909 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:23:20,910 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 949a2f97f8cb299ab38ebab18f920edc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11798472960, jitterRate=0.09881842136383057}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:23:20,910 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:20,910 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 949a2f97f8cb299ab38ebab18f920edc: Running coprocessor pre-open hook at 1732422200893Writing region info on filesystem at 1732422200893Initializing all the Stores at 1732422200894 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422200894Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422200894Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422200894Cleaning up temporary data from old regions at 1732422200906 (+12 ms)Running coprocessor post-open hooks at 1732422200910 (+4 ms)Region opened successfully at 1732422200910 2024-11-24T04:23:20,911 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc., pid=65, masterSystemTime=1732422200856 2024-11-24T04:23:20,913 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. 2024-11-24T04:23:20,913 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. 2024-11-24T04:23:20,913 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=949a2f97f8cb299ab38ebab18f920edc, regionState=OPEN, openSeqNum=2, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:20,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 949a2f97f8cb299ab38ebab18f920edc, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:23:20,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=63 2024-11-24T04:23:20,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 949a2f97f8cb299ab38ebab18f920edc, server=e57b7116ff5e,36347,1732422095008 in 211 msec 2024-11-24T04:23:20,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=61 2024-11-24T04:23:20,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=949a2f97f8cb299ab38ebab18f920edc, ASSIGN in 375 msec 2024-11-24T04:23:20,919 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T04:23:20,920 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422200919"}]},"ts":"1732422200919"} 2024-11-24T04:23:20,921 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-24T04:23:20,922 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T04:23:20,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 1.2230 sec 2024-11-24T04:23:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T04:23:21,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-24T04:23:21,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T04:23:21,864 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594 from hbase tables test-1732422103966 to tables table1 2024-11-24T04:23:21,864 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore test-1732422103966 into table1 2024-11-24T04:23:21,868 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422201865 from hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:23:21,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7316c8d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:21,869 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:21,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:21,870 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:21,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:21,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:21,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3db370a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:21,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:21,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:21,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:21,872 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:21,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3afe41cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:21,873 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:21,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:21,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5d337c10 2024-11-24T04:23:21,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:21,875 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:21,876 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-24T04:23:21,876 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table table1 2024-11-24T04:23:21,876 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T04:23:21,877 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:21,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:21,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:21,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:21,881 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: 2024-11-24T04:23:21,881 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: row0 2024-11-24T04:23:21,881 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-24T04:23:21,881 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/partitions_068ba857-7b7e-47ea-8933-b2862e8d29cf 2024-11-24T04:23:21,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741973_1149 (size=177) 2024-11-24T04:23:22,293 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:22,294 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:22,295 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,165 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-5176624230070652768.jar 2024-11-24T04:23:23,165 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,165 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,165 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,166 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,166 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,166 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,166 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,166 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,166 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,166 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:23:23,167 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T04:23:23,167 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T04:23:23,167 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T04:23:23,167 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T04:23:23,167 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T04:23:23,167 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T04:23:23,168 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T04:23:23,168 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T04:23:23,168 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T04:23:23,168 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T04:23:23,170 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,170 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,170 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,170 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,170 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,171 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:23,171 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:23:23,171 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: table1 2024-11-24T04:23:23,171 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:23:23,171 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:192) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:23,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:23,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:23,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:23,171 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-11-24T04:23:23,172 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:23:23,182 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:23:23,192 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-24T04:23:23,272 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local2106667677_0012_67bcbb46-152e-489a-a048-7baa0f72834a/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-24T04:23:23,272 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local2106667677_0012_67bcbb46-152e-489a-a048-7baa0f72834a/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-24T04:23:23,323 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3 2024-11-24T04:23:23,327 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T04:23:23,362 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/682085848b1a4dc1a2b01cad48fbef3b 2024-11-24T04:23:23,365 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T04:23:23,385 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f3/f9440cbd308f4fa3b08696b88b8a8b86 2024-11-24T04:23:23,388 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T04:23:23,417 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@230a6a25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:23,417 DEBUG [pool-267-thread-1 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:23,417 DEBUG [pool-267-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:23,418 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:23,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:23,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:23,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a2aabd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:23,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:23,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:23,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:23,419 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:23,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4561cba3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:23,420 DEBUG [pool-267-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:23,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:23,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:23,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:23,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1732422103966-3-0', locateType=CURRENT is [region=table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:23,424 INFO [pool-267-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-267-thread-1. 2024-11-24T04:23:23,424 DEBUG [pool-267-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:23,424 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:23,424 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:23,424 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:23,424 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732422103966-3-0] 2024-11-24T04:23:23,425 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: e57b7116ff5e 2024-11-24T04:23:23,429 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1376bc96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:23,429 DEBUG [pool-267-thread-1 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:23,430 DEBUG [pool-267-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:23,430 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:23,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:23,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:23,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48650935, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:23,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:23,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:23,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:23,432 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:23,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5914a9ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:23,433 DEBUG [pool-267-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:23,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:23,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:23,435 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:23,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1732422103966-3-0', locateType=CURRENT is [region=table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:23,437 INFO [pool-267-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-267-thread-1. 2024-11-24T04:23:23,437 DEBUG [pool-267-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:23,437 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:23,437 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:23,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:23,437 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732422103966-3-0] 2024-11-24T04:23:23,437 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: e57b7116ff5e 2024-11-24T04:23:23,441 DEBUG [pool-267-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422201865/_temporary/0/_temporary/attempt_local2106667677_0012_r_000000_0/f3/d7e39b8e4fb24886b9011a267c20de44 is 58, key is row-test-1732422103966-3-0/f3:q1/1732422115181/Put/seqid=0 2024-11-24T04:23:23,443 WARN [Thread-1632 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [e57b7116ff5e:36347] Specified favored nodes: [e57b7116ff5e:36347] 2024-11-24T04:23:23,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741974_1150 (size=5530) 2024-11-24T04:23:23,778 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2024-11-24T04:23:23,778 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2024-11-24T04:23:23,849 DEBUG [pool-267-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422201865/_temporary/0/_temporary/attempt_local2106667677_0012_r_000000_0/mob/1ceae8d9d10842a48540380c2f242f20 is 59, key is row-test-1732422103966-3-0/mob:q1/1732422115197/Put/seqid=0 2024-11-24T04:23:23,854 WARN [Thread-1634 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [e57b7116ff5e:36347] Specified favored nodes: [e57b7116ff5e:36347] 2024-11-24T04:23:23,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741975_1151 (size=5539) 2024-11-24T04:23:24,291 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b2e5b0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:24,291 DEBUG [pool-267-thread-1 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:24,291 DEBUG [pool-267-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:24,292 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:24,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:24,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:24,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af3a227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:24,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:24,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:24,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:24,293 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:24,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41dbcb14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:24,294 DEBUG [pool-267-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:24,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:24,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:24,297 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:24,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:24,298 INFO [pool-267-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-267-thread-1. 2024-11-24T04:23:24,298 DEBUG [pool-267-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:24,299 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:24,299 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:24,299 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:24,299 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row0] 2024-11-24T04:23:24,299 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: e57b7116ff5e 2024-11-24T04:23:24,307 DEBUG [pool-267-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422201865/_temporary/0/_temporary/attempt_local2106667677_0012_r_000001_0/f/662395b5637d42c3b575bb0bebbe9fdb is 37, key is row10/f:q1/1732422106681/Put/seqid=0 2024-11-24T04:23:24,309 WARN [Thread-1649 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [e57b7116ff5e:36347] Specified favored nodes: [e57b7116ff5e:36347] 2024-11-24T04:23:24,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741976_1152 (size=8742) 2024-11-24T04:23:25,321 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422201865 2024-11-24T04:23:25,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58283b07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,322 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:25,322 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:25,323 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:25,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:25,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:25,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5687412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:25,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:25,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,324 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:25,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d49285b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,325 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:25,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:25,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:25,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:25,330 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422201865/_SUCCESS 2024-11-24T04:23:25,338 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:25,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:25,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d317f56 2024-11-24T04:23:25,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:25,340 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:25,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:25,350 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T04:23:25,360 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422201865/f/662395b5637d42c3b575bb0bebbe9fdb first=Optional[row0] last=Optional[row98] 2024-11-24T04:23:25,363 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422201865/f3/d7e39b8e4fb24886b9011a267c20de44 first=Optional[row-test-1732422103966-3-0] last=Optional[row-test-1732422103966-3-5] 2024-11-24T04:23:25,366 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422201865/mob/1ceae8d9d10842a48540380c2f242f20 first=Optional[row-test-1732422103966-3-0] last=Optional[row-test-1732422103966-3-5] 2024-11-24T04:23:25,382 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422201865/f/662395b5637d42c3b575bb0bebbe9fdb for inclusion in 949a2f97f8cb299ab38ebab18f920edc/f 2024-11-24T04:23:25,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422201865/f3/d7e39b8e4fb24886b9011a267c20de44 for inclusion in a039ab1cc3e38eda24eec1b755eec11e/f3 2024-11-24T04:23:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(626): HFile bounds: first=row-test-1732422103966-3-0 last=row-test-1732422103966-3-5 2024-11-24T04:23:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HStore(626): HFile bounds: first=row0 last=row98 2024-11-24T04:23:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-11-24T04:23:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HStore(628): Region bounds: first=row0 last= 2024-11-24T04:23:25,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422201865/mob/1ceae8d9d10842a48540380c2f242f20 for inclusion in a039ab1cc3e38eda24eec1b755eec11e/mob 2024-11-24T04:23:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HRegion(2603): Flush status journal for 949a2f97f8cb299ab38ebab18f920edc: 2024-11-24T04:23:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422201865/f/662395b5637d42c3b575bb0bebbe9fdb to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__v87nte4selqj0jccd1f7fc1o0o67pf832ltierc9athh6jaevl2o7mkg5i68ona1/f/662395b5637d42c3b575bb0bebbe9fdb 2024-11-24T04:23:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(626): HFile bounds: first=row-test-1732422103966-3-0 last=row-test-1732422103966-3-5 2024-11-24T04:23:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-11-24T04:23:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HRegion(2603): Flush status journal for a039ab1cc3e38eda24eec1b755eec11e: 2024-11-24T04:23:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__v87nte4selqj0jccd1f7fc1o0o67pf832ltierc9athh6jaevl2o7mkg5i68ona1/f/662395b5637d42c3b575bb0bebbe9fdb as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc/f/a499fb2ba9e1418b98749dd8aa1e5bfe_SeqId_4_ 2024-11-24T04:23:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422201865/f3/d7e39b8e4fb24886b9011a267c20de44 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__v87nte4selqj0jccd1f7fc1o0o67pf832ltierc9athh6jaevl2o7mkg5i68ona1/f3/d7e39b8e4fb24886b9011a267c20de44 2024-11-24T04:23:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d1efab5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:25,395 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:25,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:25,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:25,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2129a4a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:25,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:25,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__v87nte4selqj0jccd1f7fc1o0o67pf832ltierc9athh6jaevl2o7mkg5i68ona1/f3/d7e39b8e4fb24886b9011a267c20de44 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/f3/ea1fb81ec993405a8e05a2905f851ebd_SeqId_4_ 2024-11-24T04:23:25,396 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422201865/mob/1ceae8d9d10842a48540380c2f242f20 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__v87nte4selqj0jccd1f7fc1o0o67pf832ltierc9athh6jaevl2o7mkg5i68ona1/mob/1ceae8d9d10842a48540380c2f242f20 2024-11-24T04:23:25,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c87dbea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:25,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:25,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2ae4e09d 2024-11-24T04:23:25,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__v87nte4selqj0jccd1f7fc1o0o67pf832ltierc9athh6jaevl2o7mkg5i68ona1/mob/1ceae8d9d10842a48540380c2f242f20 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/mob/7eda785d7b7c44518915e29734a54f0d_SeqId_4_ 2024-11-24T04:23:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2067373b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:25,399 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:25,400 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:25,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:25,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:25,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f6355db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:25,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:25,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:25,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:25,402 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:25,402 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:25,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2623ee41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:25,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:25,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@386ad1e3 2024-11-24T04:23:25,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:25,405 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:25,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:25,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:25,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:25,408 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:25,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:25,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:25,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:25,419 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347. 2024-11-24T04:23:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:25,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:25,420 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__v87nte4selqj0jccd1f7fc1o0o67pf832ltierc9athh6jaevl2o7mkg5i68ona1/f/662395b5637d42c3b575bb0bebbe9fdb into 949a2f97f8cb299ab38ebab18f920edc/f as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc/f/a499fb2ba9e1418b98749dd8aa1e5bfe_SeqId_4_ - updating store file list. 2024-11-24T04:23:25,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:25,426 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc/f/a499fb2ba9e1418b98749dd8aa1e5bfe_SeqId_4_ into 949a2f97f8cb299ab38ebab18f920edc/f 2024-11-24T04:23:25,426 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__v87nte4selqj0jccd1f7fc1o0o67pf832ltierc9athh6jaevl2o7mkg5i68ona1/f/662395b5637d42c3b575bb0bebbe9fdb into 949a2f97f8cb299ab38ebab18f920edc/f (new location: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc/f/a499fb2ba9e1418b98749dd8aa1e5bfe_SeqId_4_) 2024-11-24T04:23:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__v87nte4selqj0jccd1f7fc1o0o67pf832ltierc9athh6jaevl2o7mkg5i68ona1/f/662395b5637d42c3b575bb0bebbe9fdb 2024-11-24T04:23:25,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347. 2024-11-24T04:23:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,427 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@342568e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:25,428 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:25,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:25,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:25,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2abbfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:25,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:25,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,429 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:25,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51e84fc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46104096, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:25,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:25,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3845eddd 2024-11-24T04:23:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:25,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:25,431 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:25,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:25,432 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:25,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:25,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d4b53b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:25,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:25,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:25,433 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:25,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2605c2a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:25,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:25,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:25,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:25,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@66cd5f47 2024-11-24T04:23:25,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:25,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:25,435 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T04:23:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:25,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:25,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:25,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T04:23:25,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:25,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:25,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:25,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:25,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:25,450 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347. 2024-11-24T04:23:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__v87nte4selqj0jccd1f7fc1o0o67pf832ltierc9athh6jaevl2o7mkg5i68ona1/f3/d7e39b8e4fb24886b9011a267c20de44 into a039ab1cc3e38eda24eec1b755eec11e/f3 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/f3/ea1fb81ec993405a8e05a2905f851ebd_SeqId_4_ - updating store file list. 2024-11-24T04:23:25,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:25,455 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347. 2024-11-24T04:23:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:25,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/f3/ea1fb81ec993405a8e05a2905f851ebd_SeqId_4_ into a039ab1cc3e38eda24eec1b755eec11e/f3 2024-11-24T04:23:25,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__v87nte4selqj0jccd1f7fc1o0o67pf832ltierc9athh6jaevl2o7mkg5i68ona1/f3/d7e39b8e4fb24886b9011a267c20de44 into a039ab1cc3e38eda24eec1b755eec11e/f3 (new location: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/f3/ea1fb81ec993405a8e05a2905f851ebd_SeqId_4_) 2024-11-24T04:23:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__v87nte4selqj0jccd1f7fc1o0o67pf832ltierc9athh6jaevl2o7mkg5i68ona1/f3/d7e39b8e4fb24886b9011a267c20de44 2024-11-24T04:23:25,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__v87nte4selqj0jccd1f7fc1o0o67pf832ltierc9athh6jaevl2o7mkg5i68ona1/mob/1ceae8d9d10842a48540380c2f242f20 into a039ab1cc3e38eda24eec1b755eec11e/mob as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/mob/7eda785d7b7c44518915e29734a54f0d_SeqId_4_ - updating store file list. 2024-11-24T04:23:25,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/mob/7eda785d7b7c44518915e29734a54f0d_SeqId_4_ into a039ab1cc3e38eda24eec1b755eec11e/mob 2024-11-24T04:23:25,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__v87nte4selqj0jccd1f7fc1o0o67pf832ltierc9athh6jaevl2o7mkg5i68ona1/mob/1ceae8d9d10842a48540380c2f242f20 into a039ab1cc3e38eda24eec1b755eec11e/mob (new location: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/mob/7eda785d7b7c44518915e29734a54f0d_SeqId_4_) 2024-11-24T04:23:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__v87nte4selqj0jccd1f7fc1o0o67pf832ltierc9athh6jaevl2o7mkg5i68ona1/mob/1ceae8d9d10842a48540380c2f242f20 2024-11-24T04:23:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad689b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:25,465 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:25,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:25,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:25,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78d37ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:25,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:25,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,467 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:25,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695456fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:25,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:25,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d27aca1 2024-11-24T04:23:25,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:25,469 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T04:23:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:25,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:25,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:25,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T04:23:25,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:25,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:25,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:25,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347. 2024-11-24T04:23:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:25,493 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:23:25,493 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:192) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:25,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:25,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:25,494 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-11-24T04:23:25,494 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-11-24T04:23:25,494 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1732422115214 hdfs://localhost:45347/backupUT/backup_1732422115214/default/test-1732422103966/ 2024-11-24T04:23:25,495 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45347/backupUT/backup_1732422115214/.backup.manifest 2024-11-24T04:23:25,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 , the old value is region=table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=e57b7116ff5e:36347 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T04:23:25,496 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-24T04:23:25,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T04:23:25,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 from cache 2024-11-24T04:23:25,497 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732422115214/.backup.manifest 2024-11-24T04:23:25,497 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'ns2:test-17324221039661' to 'ns2:table2' from full backup image hdfs://localhost:45347/backupUT/backup_1732422115214/ns2/test-17324221039661 2024-11-24T04:23:25,508 INFO [Time-limited test {}] util.RestoreTool(491): Creating target table 'ns2:table2' 2024-11-24T04:23:25,508 DEBUG [Time-limited test {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:45347/backupUT/backup_1732422115214/ns2/test-17324221039661/archive/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1 2024-11-24T04:23:25,509 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:45347/backupUT/backup_1732422115214/ns2/test-17324221039661/archive/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f in region [hdfs://localhost:45347/backupUT/backup_1732422115214/ns2/test-17324221039661/archive/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1] 2024-11-24T04:23:25,513 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:45347/backupUT/backup_1732422115214/ns2/test-17324221039661/archive/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/933ab35b68994dc8be3ccc706f0c0aa9 first=row0 last=row98 2024-11-24T04:23:25,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:23:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:table2 2024-11-24T04:23:25,517 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T04:23:25,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 66 2024-11-24T04:23:25,518 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T04:23:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T04:23:25,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741977_1153 (size=351) 2024-11-24T04:23:25,602 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'table1', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=e57b7116ff5e:36347 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 34 more 2024-11-24T04:23:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T04:23:25,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T04:23:25,926 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 589b5c4b38f159789612c7e3246e2779, NAME => 'ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:23:25,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741978_1154 (size=37) 2024-11-24T04:23:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T04:23:26,332 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:23:26,332 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1722): Closing 589b5c4b38f159789612c7e3246e2779, disabling compactions & flushes 2024-11-24T04:23:26,332 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. 2024-11-24T04:23:26,333 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. 2024-11-24T04:23:26,333 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. after waiting 0 ms 2024-11-24T04:23:26,333 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. 2024-11-24T04:23:26,333 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1973): Closed ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. 2024-11-24T04:23:26,333 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1676): Region close journal for 589b5c4b38f159789612c7e3246e2779: Waiting for close lock at 1732422206332Disabling compacts and flushes for region at 1732422206332Disabling writes for close at 1732422206333 (+1 ms)Writing region close event to WAL at 1732422206333Closed at 1732422206333 2024-11-24T04:23:26,334 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T04:23:26,335 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1732422206334"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422206334"}]},"ts":"1732422206334"} 2024-11-24T04:23:26,338 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T04:23:26,340 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T04:23:26,340 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422206340"}]},"ts":"1732422206340"} 2024-11-24T04:23:26,343 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2024-11-24T04:23:26,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=589b5c4b38f159789612c7e3246e2779, ASSIGN}] 2024-11-24T04:23:26,346 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=589b5c4b38f159789612c7e3246e2779, ASSIGN 2024-11-24T04:23:26,347 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=589b5c4b38f159789612c7e3246e2779, ASSIGN; state=OFFLINE, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=false 2024-11-24T04:23:26,400 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-24T04:23:26,429 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-24T04:23:26,498 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=589b5c4b38f159789612c7e3246e2779, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:26,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=589b5c4b38f159789612c7e3246e2779, ASSIGN because future has completed 2024-11-24T04:23:26,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 589b5c4b38f159789612c7e3246e2779, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:23:26,547 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5a030641752b7def7c38c5f6295501b2, had cached 0 bytes from a total of 41499 2024-11-24T04:23:26,659 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(132): Open ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. 2024-11-24T04:23:26,659 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7752): Opening region: {ENCODED => 589b5c4b38f159789612c7e3246e2779, NAME => 'ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:23:26,660 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:23:26,660 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 589b5c4b38f159789612c7e3246e2779 2024-11-24T04:23:26,661 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(898): Instantiated ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:23:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T04:23:26,661 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7794): checking encryption for 589b5c4b38f159789612c7e3246e2779 2024-11-24T04:23:26,661 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7797): checking classloading for 589b5c4b38f159789612c7e3246e2779 2024-11-24T04:23:26,664 INFO [StoreOpener-589b5c4b38f159789612c7e3246e2779-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 589b5c4b38f159789612c7e3246e2779 2024-11-24T04:23:26,666 INFO [StoreOpener-589b5c4b38f159789612c7e3246e2779-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 589b5c4b38f159789612c7e3246e2779 columnFamilyName f 2024-11-24T04:23:26,666 DEBUG [StoreOpener-589b5c4b38f159789612c7e3246e2779-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:26,666 INFO [StoreOpener-589b5c4b38f159789612c7e3246e2779-1 {}] regionserver.HStore(327): Store=589b5c4b38f159789612c7e3246e2779/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:26,667 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1038): replaying wal for 589b5c4b38f159789612c7e3246e2779 2024-11-24T04:23:26,668 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/589b5c4b38f159789612c7e3246e2779 2024-11-24T04:23:26,668 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/589b5c4b38f159789612c7e3246e2779 2024-11-24T04:23:26,668 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1048): stopping wal replay for 589b5c4b38f159789612c7e3246e2779 2024-11-24T04:23:26,669 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1060): Cleaning up temporary data for 589b5c4b38f159789612c7e3246e2779 2024-11-24T04:23:26,671 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1093): writing seq id for 589b5c4b38f159789612c7e3246e2779 2024-11-24T04:23:26,673 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/589b5c4b38f159789612c7e3246e2779/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:23:26,674 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1114): Opened 589b5c4b38f159789612c7e3246e2779; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10803809120, jitterRate=0.006183132529258728}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:23:26,674 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 589b5c4b38f159789612c7e3246e2779 2024-11-24T04:23:26,675 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1006): Region open journal for 589b5c4b38f159789612c7e3246e2779: Running coprocessor pre-open hook at 1732422206661Writing region info on filesystem at 1732422206661Initializing all the Stores at 1732422206663 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422206663Cleaning up temporary data from old regions at 1732422206669 (+6 ms)Running coprocessor post-open hooks at 1732422206674 (+5 ms)Region opened successfully at 1732422206675 (+1 ms) 2024-11-24T04:23:26,676 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779., pid=68, masterSystemTime=1732422206652 2024-11-24T04:23:26,678 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. 2024-11-24T04:23:26,678 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(153): Opened ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. 2024-11-24T04:23:26,679 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=589b5c4b38f159789612c7e3246e2779, regionState=OPEN, openSeqNum=2, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:26,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 589b5c4b38f159789612c7e3246e2779, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:23:26,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-24T04:23:26,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 589b5c4b38f159789612c7e3246e2779, server=e57b7116ff5e,36347,1732422095008 in 181 msec 2024-11-24T04:23:26,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-24T04:23:26,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=589b5c4b38f159789612c7e3246e2779, ASSIGN in 339 msec 2024-11-24T04:23:26,686 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T04:23:26,686 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422206686"}]},"ts":"1732422206686"} 2024-11-24T04:23:26,687 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2024-11-24T04:23:26,688 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T04:23:26,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:table2 in 1.1740 sec 2024-11-24T04:23:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T04:23:27,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:table2 completed 2024-11-24T04:23:27,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-24T04:23:27,682 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45347/backupUT/backup_1732422115214/ns2/test-17324221039661/archive/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1 from hbase tables ns2:test-17324221039661 to tables ns2:table2 2024-11-24T04:23:27,682 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-17324221039661 into ns2:table2 2024-11-24T04:23:27,684 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-ns2-table2-1732422207682 from hdfs://localhost:45347/backupUT/backup_1732422115214/ns2/test-17324221039661/archive/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1 2024-11-24T04:23:27,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61bbe09b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:27,684 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:27,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:27,685 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:27,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:27,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:27,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103d4a17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:27,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:27,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:27,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:27,687 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:27,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@305e3b7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:27,688 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:27,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:27,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5d0f0ca5 2024-11-24T04:23:27,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:27,689 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:27,690 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-24T04:23:27,690 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:table2 2024-11-24T04:23:27,690 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-24T04:23:27,690 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:27,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:27,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:27,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:27,694 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:table2: 2024-11-24T04:23:27,694 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 1 reduce partitions to match current region count for all tables 2024-11-24T04:23:27,694 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/partitions_fc9d1468-ceb1-43c6-a6fa-a7eed0c2cd2a 2024-11-24T04:23:27,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741979_1155 (size=153) 2024-11-24T04:23:28,103 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,104 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,104 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,966 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-9252758867290173272.jar 2024-11-24T04:23:28,966 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,966 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,967 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,967 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,967 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,967 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,967 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,967 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,968 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,968 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:23:28,968 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T04:23:28,968 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T04:23:28,968 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T04:23:28,968 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T04:23:28,969 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T04:23:28,969 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T04:23:28,969 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T04:23:28,969 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T04:23:28,969 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T04:23:28,969 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T04:23:28,970 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,970 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,970 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,970 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,970 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,970 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:28,970 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:23:28,971 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:table2 2024-11-24T04:23:28,971 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:23:28,971 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:192) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:28,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:28,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:28,971 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:28,971 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-11-24T04:23:28,971 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:23:28,980 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:23:28,992 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-24T04:23:29,076 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local379687726_0013_74ddfa7f-b31b-463d-b732-67465ac27d5c/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-24T04:23:29,076 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local379687726_0013_74ddfa7f-b31b-463d-b732-67465ac27d5c/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-24T04:23:29,128 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45347/backupUT/backup_1732422115214/ns2/test-17324221039661/archive/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/933ab35b68994dc8be3ccc706f0c0aa9 2024-11-24T04:23:29,132 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T04:23:29,162 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eedfd2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:29,162 DEBUG [pool-272-thread-1 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:29,162 DEBUG [pool-272-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:29,163 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:29,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:29,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:29,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e353188, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:29,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:29,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:29,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:29,166 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:29,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c7f31f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:29,167 DEBUG [pool-272-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:29,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:29,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:29,170 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:29,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='row0', locateType=CURRENT is [region=ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:29,171 INFO [pool-272-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-272-thread-1. 2024-11-24T04:23:29,171 DEBUG [pool-272-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:29,171 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:29,172 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:29,172 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:29,172 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row0] 2024-11-24T04:23:29,172 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: e57b7116ff5e 2024-11-24T04:23:29,179 DEBUG [pool-272-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-ns2-table2-1732422207682/_temporary/0/_temporary/attempt_local379687726_0013_r_000000_0/f/fbc2119601eb4fc9ad0e0e15c005e906 is 37, key is row10/f:q1/1732422109020/Put/seqid=0 2024-11-24T04:23:29,180 WARN [Thread-1716 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [e57b7116ff5e:36347] Specified favored nodes: [e57b7116ff5e:36347] 2024-11-24T04:23:29,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741980_1156 (size=8741) 2024-11-24T04:23:30,121 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-ns2-table2-1732422207682 2024-11-24T04:23:30,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6615624a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:30,122 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:30,122 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:30,123 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:30,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:30,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:30,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aab06f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:30,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:30,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:30,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:30,124 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:30,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3204c076, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:30,125 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:30,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:30,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:30,127 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:30,129 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-ns2-table2-1732422207682/_SUCCESS 2024-11-24T04:23:30,132 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:30,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:30,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@31dc129c 2024-11-24T04:23:30,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:30,135 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:30,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:30,141 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-24T04:23:30,147 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-ns2-table2-1732422207682/f/fbc2119601eb4fc9ad0e0e15c005e906 first=Optional[row0] last=Optional[row98] 2024-11-24T04:23:30,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-ns2-table2-1732422207682/f/fbc2119601eb4fc9ad0e0e15c005e906 for inclusion in 589b5c4b38f159789612c7e3246e2779/f 2024-11-24T04:23:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(626): HFile bounds: first=row0 last=row98 2024-11-24T04:23:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T04:23:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HRegion(2603): Flush status journal for 589b5c4b38f159789612c7e3246e2779: 2024-11-24T04:23:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-ns2-table2-1732422207682/f/fbc2119601eb4fc9ad0e0e15c005e906 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__ns2_table2__83ojpshk8kii11oh6ct8n2fs8g9b2mjhopq2o1dqejffiedmf5ectq79oitfrr99/f/fbc2119601eb4fc9ad0e0e15c005e906 2024-11-24T04:23:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__ns2_table2__83ojpshk8kii11oh6ct8n2fs8g9b2mjhopq2o1dqejffiedmf5ectq79oitfrr99/f/fbc2119601eb4fc9ad0e0e15c005e906 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/589b5c4b38f159789612c7e3246e2779/f/e8a8d2160ae7448882738d7c31ecdf11_SeqId_4_ 2024-11-24T04:23:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cfeeedb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:30,159 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:30,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:30,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:30,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2528b633, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:30,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:30,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:30,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:30,161 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:30,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ae6ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:30,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:30,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1ee8d9b8 2024-11-24T04:23:30,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:30,164 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:30,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:30,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:30,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:30,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:30,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:30,183 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:30,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347. 2024-11-24T04:23:30,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:30,187 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:30,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__ns2_table2__83ojpshk8kii11oh6ct8n2fs8g9b2mjhopq2o1dqejffiedmf5ectq79oitfrr99/f/fbc2119601eb4fc9ad0e0e15c005e906 into 589b5c4b38f159789612c7e3246e2779/f as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/589b5c4b38f159789612c7e3246e2779/f/e8a8d2160ae7448882738d7c31ecdf11_SeqId_4_ - updating store file list. 2024-11-24T04:23:30,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/589b5c4b38f159789612c7e3246e2779/f/e8a8d2160ae7448882738d7c31ecdf11_SeqId_4_ into 589b5c4b38f159789612c7e3246e2779/f 2024-11-24T04:23:30,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__ns2_table2__83ojpshk8kii11oh6ct8n2fs8g9b2mjhopq2o1dqejffiedmf5ectq79oitfrr99/f/fbc2119601eb4fc9ad0e0e15c005e906 into 589b5c4b38f159789612c7e3246e2779/f (new location: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/589b5c4b38f159789612c7e3246e2779/f/e8a8d2160ae7448882738d7c31ecdf11_SeqId_4_) 2024-11-24T04:23:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__ns2_table2__83ojpshk8kii11oh6ct8n2fs8g9b2mjhopq2o1dqejffiedmf5ectq79oitfrr99/f/fbc2119601eb4fc9ad0e0e15c005e906 2024-11-24T04:23:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@97122ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:30,195 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:30,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:30,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:30,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70f5d4bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:30,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:30,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:30,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:30,196 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:30,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77556d61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:30,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:30,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@76206130 2024-11-24T04:23:30,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:30,198 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T04:23:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:30,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:30,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:30,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T04:23:30,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:30,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:30,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:30,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347. 2024-11-24T04:23:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:30,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:30,217 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:23:30,217 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:192) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:30,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:30,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:30,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:30,217 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-11-24T04:23:30,217 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-11-24T04:23:30,217 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1732422115214 hdfs://localhost:45347/backupUT/backup_1732422115214/ns2/test-17324221039661/ 2024-11-24T04:23:30,218 DEBUG [Time-limited test {}] impl.RestoreTablesClient(236): restoreStage finished 2024-11-24T04:23:30,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 , the old value is region=ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=e57b7116ff5e:36347 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T04:23:30,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T04:23:30,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 from cache 2024-11-24T04:23:30,218 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-24T04:23:30,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:30,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:30,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:30,245 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45347/backupUT/backup_1732422186353/.backup.manifest 2024-11-24T04:23:30,248 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732422186353/.backup.manifest 2024-11-24T04:23:30,249 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45347/backupUT/backup_1732422151095/.backup.manifest 2024-11-24T04:23:30,251 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732422151095/.backup.manifest 2024-11-24T04:23:30,251 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'test-1732422103966' to 'table1' from full backup image hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966 2024-11-24T04:23:30,254 DEBUG [Time-limited test {}] util.RestoreTool(295): Retrieved descriptor: 'test-1732422103966', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} thru backup_1732422186353 2024-11-24T04:23:30,258 INFO [Time-limited test {}] util.RestoreTool(480): Truncating exising target table 'table1', preserving region splits 2024-11-24T04:23:30,263 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable table1 2024-11-24T04:23:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=69, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=table1 2024-11-24T04:23:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-11-24T04:23:30,270 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422210270"}]},"ts":"1732422210270"} 2024-11-24T04:23:30,272 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=DISABLING in hbase:meta 2024-11-24T04:23:30,272 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set table1 to state=DISABLING 2024-11-24T04:23:30,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=table1}] 2024-11-24T04:23:30,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=a039ab1cc3e38eda24eec1b755eec11e, UNASSIGN}, {pid=72, ppid=70, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=949a2f97f8cb299ab38ebab18f920edc, UNASSIGN}] 2024-11-24T04:23:30,277 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=70, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=949a2f97f8cb299ab38ebab18f920edc, UNASSIGN 2024-11-24T04:23:30,277 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=70, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=a039ab1cc3e38eda24eec1b755eec11e, UNASSIGN 2024-11-24T04:23:30,278 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=949a2f97f8cb299ab38ebab18f920edc, regionState=CLOSING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:30,278 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=a039ab1cc3e38eda24eec1b755eec11e, regionState=CLOSING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:30,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=70, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=949a2f97f8cb299ab38ebab18f920edc, UNASSIGN because future has completed 2024-11-24T04:23:30,281 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T04:23:30,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure 949a2f97f8cb299ab38ebab18f920edc, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:23:30,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=70, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=a039ab1cc3e38eda24eec1b755eec11e, UNASSIGN because future has completed 2024-11-24T04:23:30,282 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T04:23:30,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=71, state=RUNNABLE, hasLock=false; CloseRegionProcedure a039ab1cc3e38eda24eec1b755eec11e, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:23:30,321 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'ns2:table2', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=e57b7116ff5e:36347 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 34 more 2024-11-24T04:23:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-11-24T04:23:30,434 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] handler.UnassignRegionHandler(122): Close 949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:30,434 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T04:23:30,434 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1722): Closing 949a2f97f8cb299ab38ebab18f920edc, disabling compactions & flushes 2024-11-24T04:23:30,434 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1755): Closing region table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. 2024-11-24T04:23:30,434 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. 2024-11-24T04:23:30,434 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. after waiting 0 ms 2024-11-24T04:23:30,434 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. 2024-11-24T04:23:30,442 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-24T04:23:30,443 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:23:30,444 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1973): Closed table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. 2024-11-24T04:23:30,444 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1676): Region close journal for 949a2f97f8cb299ab38ebab18f920edc: Waiting for close lock at 1732422210434Running coprocessor pre-close hooks at 1732422210434Disabling compacts and flushes for region at 1732422210434Disabling writes for close at 1732422210434Writing region close event to WAL at 1732422210436 (+2 ms)Running coprocessor post-close hooks at 1732422210443 (+7 ms)Closed at 1732422210444 (+1 ms) 2024-11-24T04:23:30,447 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] handler.UnassignRegionHandler(157): Closed 949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:30,447 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] handler.UnassignRegionHandler(122): Close a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:30,447 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T04:23:30,447 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1722): Closing a039ab1cc3e38eda24eec1b755eec11e, disabling compactions & flushes 2024-11-24T04:23:30,448 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1755): Closing region table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. 2024-11-24T04:23:30,448 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. 2024-11-24T04:23:30,448 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1843): Acquired close lock on table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. after waiting 0 ms 2024-11-24T04:23:30,448 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1853): Updates disabled for region table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. 2024-11-24T04:23:30,448 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=949a2f97f8cb299ab38ebab18f920edc, regionState=CLOSED 2024-11-24T04:23:30,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure 949a2f97f8cb299ab38ebab18f920edc, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:23:30,453 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-24T04:23:30,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-24T04:23:30,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; CloseRegionProcedure 949a2f97f8cb299ab38ebab18f920edc, server=e57b7116ff5e,36347,1732422095008 in 170 msec 2024-11-24T04:23:30,454 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:23:30,454 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1973): Closed table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. 2024-11-24T04:23:30,454 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1676): Region close journal for a039ab1cc3e38eda24eec1b755eec11e: Waiting for close lock at 1732422210447Running coprocessor pre-close hooks at 1732422210447Disabling compacts and flushes for region at 1732422210447Disabling writes for close at 1732422210448 (+1 ms)Writing region close event to WAL at 1732422210449 (+1 ms)Running coprocessor post-close hooks at 1732422210454 (+5 ms)Closed at 1732422210454 2024-11-24T04:23:30,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=70, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=949a2f97f8cb299ab38ebab18f920edc, UNASSIGN in 178 msec 2024-11-24T04:23:30,456 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] handler.UnassignRegionHandler(157): Closed a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:30,456 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=a039ab1cc3e38eda24eec1b755eec11e, regionState=CLOSED 2024-11-24T04:23:30,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=71, state=RUNNABLE, hasLock=false; CloseRegionProcedure a039ab1cc3e38eda24eec1b755eec11e, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:23:30,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=71 2024-11-24T04:23:30,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=71, state=SUCCESS, hasLock=false; CloseRegionProcedure a039ab1cc3e38eda24eec1b755eec11e, server=e57b7116ff5e,36347,1732422095008 in 177 msec 2024-11-24T04:23:30,463 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-24T04:23:30,463 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=a039ab1cc3e38eda24eec1b755eec11e, UNASSIGN in 185 msec 2024-11-24T04:23:30,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-11-24T04:23:30,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=table1 in 191 msec 2024-11-24T04:23:30,466 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422210466"}]},"ts":"1732422210466"} 2024-11-24T04:23:30,467 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=DISABLED in hbase:meta 2024-11-24T04:23:30,467 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set table1 to state=DISABLED 2024-11-24T04:23:30,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, state=SUCCESS, hasLock=false; DisableTableProcedure table=table1 in 204 msec 2024-11-24T04:23:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-11-24T04:23:30,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:table1 completed 2024-11-24T04:23:30,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.HMaster$6(2602): Client=jenkins//172.17.0.3 truncate table1 2024-11-24T04:23:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=75, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION, hasLock=false; TruncateTableProcedure (table=table1 preserveSplits=true) 2024-11-24T04:23:30,608 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(87): waiting for 'table1' regions in transition 2024-11-24T04:23:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-24T04:23:30,613 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:30,613 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:30,616 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/f, FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/f3, FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/mob, FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/recovered.edits] 2024-11-24T04:23:30,616 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc/f, FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc/f3, FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc/mob, FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc/recovered.edits] 2024-11-24T04:23:30,620 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc/f/a499fb2ba9e1418b98749dd8aa1e5bfe_SeqId_4_ to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/archive/data/default/table1/949a2f97f8cb299ab38ebab18f920edc/f/a499fb2ba9e1418b98749dd8aa1e5bfe_SeqId_4_ 2024-11-24T04:23:30,621 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/f3/ea1fb81ec993405a8e05a2905f851ebd_SeqId_4_ to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/archive/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/f3/ea1fb81ec993405a8e05a2905f851ebd_SeqId_4_ 2024-11-24T04:23:30,624 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/mob/7eda785d7b7c44518915e29734a54f0d_SeqId_4_ to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/archive/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/mob/7eda785d7b7c44518915e29734a54f0d_SeqId_4_ 2024-11-24T04:23:30,625 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc/recovered.edits/6.seqid to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/archive/data/default/table1/949a2f97f8cb299ab38ebab18f920edc/recovered.edits/6.seqid 2024-11-24T04:23:30,626 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/949a2f97f8cb299ab38ebab18f920edc 2024-11-24T04:23:30,627 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/recovered.edits/6.seqid to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/archive/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e/recovered.edits/6.seqid 2024-11-24T04:23:30,628 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a039ab1cc3e38eda24eec1b755eec11e 2024-11-24T04:23:30,628 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived table1 regions 2024-11-24T04:23:30,629 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/mobdir/data/default/table1/b4aaa492ee713125ae5609f2682a2d2f 2024-11-24T04:23:30,629 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/mobdir/data/default/table1/b4aaa492ee713125ae5609f2682a2d2f/mob] 2024-11-24T04:23:30,631 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/mobdir/data/default/table1/b4aaa492ee713125ae5609f2682a2d2f 2024-11-24T04:23:30,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36347 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-24T04:23:30,640 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 2 rows of table1 from hbase:meta 2024-11-24T04:23:30,643 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'table1' descriptor. 2024-11-24T04:23:30,644 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'table1' from region states. 2024-11-24T04:23:30,644 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732422210644"}]},"ts":"9223372036854775807"} 2024-11-24T04:23:30,644 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732422210644"}]},"ts":"9223372036854775807"} 2024-11-24T04:23:30,646 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 2 regions from META 2024-11-24T04:23:30,646 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a039ab1cc3e38eda24eec1b755eec11e, NAME => 'table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e.', STARTKEY => '', ENDKEY => 'row0'}, {ENCODED => 949a2f97f8cb299ab38ebab18f920edc, NAME => 'table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc.', STARTKEY => 'row0', ENDKEY => ''}] 2024-11-24T04:23:30,646 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'table1' as deleted. 2024-11-24T04:23:30,647 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732422210646"}]},"ts":"9223372036854775807"} 2024-11-24T04:23:30,648 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table table1 state from META 2024-11-24T04:23:30,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741981_1157 (size=938) 2024-11-24T04:23:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-24T04:23:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-24T04:23:31,061 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 69e3c311ed1f46d175b4045bb296446a, NAME => 'table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a.', STARTKEY => '', ENDKEY => 'row0'}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:23:31,063 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a15495df03a7b8d0d73077c21d437b8b, NAME => 'table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b.', STARTKEY => 'row0', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:23:31,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741982_1158 (size=45) 2024-11-24T04:23:31,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741983_1159 (size=45) 2024-11-24T04:23:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-24T04:23:31,474 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:23:31,475 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing 69e3c311ed1f46d175b4045bb296446a, disabling compactions & flushes 2024-11-24T04:23:31,475 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:31,475 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:31,475 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. after waiting 0 ms 2024-11-24T04:23:31,475 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:31,475 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:31,475 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for 69e3c311ed1f46d175b4045bb296446a: Waiting for close lock at 1732422211474Disabling compacts and flushes for region at 1732422211474Disabling writes for close at 1732422211475 (+1 ms)Writing region close event to WAL at 1732422211475Closed at 1732422211475 2024-11-24T04:23:31,478 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(898): Instantiated table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:23:31,478 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1722): Closing a15495df03a7b8d0d73077c21d437b8b, disabling compactions & flushes 2024-11-24T04:23:31,478 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1755): Closing region table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:31,478 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:31,478 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. after waiting 0 ms 2024-11-24T04:23:31,478 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:31,478 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1973): Closed table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:31,478 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1676): Region close journal for a15495df03a7b8d0d73077c21d437b8b: Waiting for close lock at 1732422211478Disabling compacts and flushes for region at 1732422211478Disabling writes for close at 1732422211478Writing region close event to WAL at 1732422211478Closed at 1732422211478 2024-11-24T04:23:31,481 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732422211481"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422211481"}]},"ts":"1732422211481"} 2024-11-24T04:23:31,482 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732422211481"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422211481"}]},"ts":"1732422211481"} 2024-11-24T04:23:31,486 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 2 regions to meta. 2024-11-24T04:23:31,488 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422211488"}]},"ts":"1732422211488"} 2024-11-24T04:23:31,491 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-24T04:23:31,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=69e3c311ed1f46d175b4045bb296446a, ASSIGN}, {pid=77, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=a15495df03a7b8d0d73077c21d437b8b, ASSIGN}] 2024-11-24T04:23:31,492 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=77, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=a15495df03a7b8d0d73077c21d437b8b, ASSIGN 2024-11-24T04:23:31,492 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=69e3c311ed1f46d175b4045bb296446a, ASSIGN 2024-11-24T04:23:31,493 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=77, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=a15495df03a7b8d0d73077c21d437b8b, ASSIGN; state=OFFLINE, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=false 2024-11-24T04:23:31,493 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=69e3c311ed1f46d175b4045bb296446a, ASSIGN; state=OFFLINE, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=false 2024-11-24T04:23:31,645 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=77 updating hbase:meta row=a15495df03a7b8d0d73077c21d437b8b, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:31,645 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=69e3c311ed1f46d175b4045bb296446a, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:31,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=a15495df03a7b8d0d73077c21d437b8b, ASSIGN because future has completed 2024-11-24T04:23:31,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE, hasLock=false; OpenRegionProcedure a15495df03a7b8d0d73077c21d437b8b, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:23:31,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=69e3c311ed1f46d175b4045bb296446a, ASSIGN because future has completed 2024-11-24T04:23:31,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=76, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69e3c311ed1f46d175b4045bb296446a, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:23:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-24T04:23:31,811 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:31,811 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 69e3c311ed1f46d175b4045bb296446a, NAME => 'table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a.', STARTKEY => '', ENDKEY => 'row0'} 2024-11-24T04:23:31,812 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:23:31,812 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:31,812 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:23:31,812 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:31,812 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:31,814 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:31,816 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69e3c311ed1f46d175b4045bb296446a columnFamilyName f 2024-11-24T04:23:31,816 DEBUG [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:31,817 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] regionserver.HStore(327): Store=69e3c311ed1f46d175b4045bb296446a/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:31,817 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:31,819 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69e3c311ed1f46d175b4045bb296446a columnFamilyName f3 2024-11-24T04:23:31,819 DEBUG [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:31,819 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] regionserver.HStore(327): Store=69e3c311ed1f46d175b4045bb296446a/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:31,819 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:31,821 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69e3c311ed1f46d175b4045bb296446a columnFamilyName mob 2024-11-24T04:23:31,822 DEBUG [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:31,823 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] regionserver.HStore(327): Store=69e3c311ed1f46d175b4045bb296446a/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:31,823 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:31,824 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:31,825 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:31,827 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:31,827 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:31,828 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:23:31,829 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:31,832 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:23:31,833 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 69e3c311ed1f46d175b4045bb296446a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11915585120, jitterRate=0.10972534120082855}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:23:31,833 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:31,833 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 69e3c311ed1f46d175b4045bb296446a: Running coprocessor pre-open hook at 1732422211812Writing region info on filesystem at 1732422211812Initializing all the Stores at 1732422211814 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422211814Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422211814Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422211814Cleaning up temporary data from old regions at 1732422211827 (+13 ms)Running coprocessor post-open hooks at 1732422211833 (+6 ms)Region opened successfully at 1732422211833 2024-11-24T04:23:31,834 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., pid=79, masterSystemTime=1732422211802 2024-11-24T04:23:31,836 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:31,836 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:31,836 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:31,836 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => a15495df03a7b8d0d73077c21d437b8b, NAME => 'table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b.', STARTKEY => 'row0', ENDKEY => ''} 2024-11-24T04:23:31,836 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=69e3c311ed1f46d175b4045bb296446a, regionState=OPEN, openSeqNum=2, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:31,836 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:23:31,836 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:31,837 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:23:31,837 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:31,837 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:31,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=76, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69e3c311ed1f46d175b4045bb296446a, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:23:31,838 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:31,839 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a15495df03a7b8d0d73077c21d437b8b columnFamilyName f 2024-11-24T04:23:31,839 DEBUG [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:31,840 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] regionserver.HStore(327): Store=a15495df03a7b8d0d73077c21d437b8b/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:31,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=76 2024-11-24T04:23:31,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=76, state=SUCCESS, hasLock=false; OpenRegionProcedure 69e3c311ed1f46d175b4045bb296446a, server=e57b7116ff5e,36347,1732422095008 in 187 msec 2024-11-24T04:23:31,840 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:31,841 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a15495df03a7b8d0d73077c21d437b8b columnFamilyName f3 2024-11-24T04:23:31,841 DEBUG [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:31,841 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=69e3c311ed1f46d175b4045bb296446a, ASSIGN in 349 msec 2024-11-24T04:23:31,841 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] regionserver.HStore(327): Store=a15495df03a7b8d0d73077c21d437b8b/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:31,841 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:31,842 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a15495df03a7b8d0d73077c21d437b8b columnFamilyName mob 2024-11-24T04:23:31,843 DEBUG [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:31,843 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] regionserver.HStore(327): Store=a15495df03a7b8d0d73077c21d437b8b/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:31,843 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:31,844 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:31,844 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:31,845 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:31,845 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:31,846 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:23:31,847 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:31,849 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a15495df03a7b8d0d73077c21d437b8b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:23:31,849 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened a15495df03a7b8d0d73077c21d437b8b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9439333440, jitterRate=-0.12089356780052185}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:23:31,849 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:31,850 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for a15495df03a7b8d0d73077c21d437b8b: Running coprocessor pre-open hook at 1732422211837Writing region info on filesystem at 1732422211837Initializing all the Stores at 1732422211838 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422211838Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422211838Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422211838Cleaning up temporary data from old regions at 1732422211845 (+7 ms)Running coprocessor post-open hooks at 1732422211850 (+5 ms)Region opened successfully at 1732422211850 2024-11-24T04:23:31,850 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b., pid=78, masterSystemTime=1732422211802 2024-11-24T04:23:31,852 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:31,852 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:31,852 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=77 updating hbase:meta row=a15495df03a7b8d0d73077c21d437b8b, regionState=OPEN, openSeqNum=2, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:31,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE, hasLock=false; OpenRegionProcedure a15495df03a7b8d0d73077c21d437b8b, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:23:31,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-24T04:23:31,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; OpenRegionProcedure a15495df03a7b8d0d73077c21d437b8b, server=e57b7116ff5e,36347,1732422095008 in 204 msec 2024-11-24T04:23:31,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=75 2024-11-24T04:23:31,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=a15495df03a7b8d0d73077c21d437b8b, ASSIGN in 365 msec 2024-11-24T04:23:31,858 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422211857"}]},"ts":"1732422211857"} 2024-11-24T04:23:31,859 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-24T04:23:31,859 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(145): truncate 'table1' completed 2024-11-24T04:23:31,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, state=SUCCESS, hasLock=false; TruncateTableProcedure (table=table1 preserveSplits=true) in 1.2570 sec 2024-11-24T04:23:32,164 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-24T04:23:32,381 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T04:23:32,431 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:table2' 2024-11-24T04:23:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-24T04:23:32,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: TRUNCATE, Table Name: default:table1 completed 2024-11-24T04:23:32,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T04:23:32,763 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594,hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/archive/data/default/test-1732422103966/3047fc58bb55744b4303d1cceb01bae9 from hbase tables test-1732422103966 to tables table1 2024-11-24T04:23:32,763 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore test-1732422103966 into table1 2024-11-24T04:23:32,765 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422212763 from hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594,hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/archive/data/default/test-1732422103966/3047fc58bb55744b4303d1cceb01bae9 2024-11-24T04:23:32,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@213d6bff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:32,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:32,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:32,767 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:32,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:32,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:32,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52a17ea0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:32,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:32,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:32,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:32,768 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:32,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a326547, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:32,769 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:32,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:32,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4170d9f9 2024-11-24T04:23:32,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:32,771 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:32,771 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-24T04:23:32,771 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table table1 2024-11-24T04:23:32,772 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T04:23:32,772 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:32,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:32,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:32,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:32,775 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: 2024-11-24T04:23:32,776 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: row0 2024-11-24T04:23:32,776 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-24T04:23:32,776 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/partitions_98790c30-6e8f-46ac-a6e9-4aeb37be1d88 2024-11-24T04:23:32,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741984_1160 (size=177) 2024-11-24T04:23:33,183 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:33,184 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:33,184 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:33,778 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2024-11-24T04:23:33,778 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_table2 Metrics about Tables on a single HBase RegionServer 2024-11-24T04:23:34,046 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-7810954796168611266.jar 2024-11-24T04:23:34,047 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:34,047 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:34,047 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:34,047 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:34,047 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:34,047 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:34,048 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:34,048 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:34,048 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:34,048 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:23:34,048 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T04:23:34,048 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T04:23:34,049 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T04:23:34,049 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T04:23:34,049 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T04:23:34,049 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T04:23:34,049 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T04:23:34,049 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T04:23:34,049 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T04:23:34,050 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T04:23:34,050 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:34,050 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:34,050 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:34,050 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:34,050 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:34,051 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:34,051 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:23:34,051 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: table1 2024-11-24T04:23:34,051 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:23:34,051 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:34,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:34,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:34,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:34,051 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-11-24T04:23:34,052 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:23:34,061 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:23:34,071 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-24T04:23:34,164 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local189592895_0014_f8a21955-80a1-44cd-a183-c00f60dad70d/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-24T04:23:34,164 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local189592895_0014_f8a21955-80a1-44cd-a183-c00f60dad70d/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-24T04:23:34,211 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/4adb0dbc87f947bdaad87e7746c93da8 2024-11-24T04:23:34,214 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T04:23:34,251 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/60684cdd8d5e45cbaabfe6505beaad9f 2024-11-24T04:23:34,254 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T04:23:34,278 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f/a73d9d62109f4ed1b1114d2cc76fdef3 2024-11-24T04:23:34,281 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T04:23:34,311 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/archive/data/default/test-1732422103966/3047fc58bb55744b4303d1cceb01bae9/mob/d41d8cd98f00b204e9800998ecf8427e20241124065620beac114f4a94d99cd6792a819b_2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:23:34,314 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T04:23:34,334 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45347/backupUT/backup_1732422151095/default/test-1732422103966/archive/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/mob/682085848b1a4dc1a2b01cad48fbef3b 2024-11-24T04:23:34,337 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T04:23:34,375 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@464a5a43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:34,375 DEBUG [pool-277-thread-1 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:34,376 DEBUG [pool-277-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:34,376 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:34,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:34,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:34,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@623133c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:34,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:34,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:34,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:34,378 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:34,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@435476f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:34,379 DEBUG [pool-277-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:34,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:34,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:34,381 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:34,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1732422103966-1-0', locateType=CURRENT is [region=table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:34,383 INFO [pool-277-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-277-thread-1. 2024-11-24T04:23:34,383 DEBUG [pool-277-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:34,383 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:34,383 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:34,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:34,383 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732422103966-1-0] 2024-11-24T04:23:34,383 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: e57b7116ff5e 2024-11-24T04:23:34,393 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55cbe738, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:34,393 DEBUG [pool-277-thread-1 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:34,394 DEBUG [pool-277-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:34,394 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:34,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:34,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:34,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31abf7b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:34,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:34,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:34,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:34,396 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:34,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@787a0de0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:34,397 DEBUG [pool-277-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:34,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:34,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:34,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:34,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1732422103966-3-0', locateType=CURRENT is [region=table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:34,401 INFO [pool-277-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-277-thread-1. 2024-11-24T04:23:34,401 DEBUG [pool-277-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:34,401 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:34,402 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:34,402 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:34,402 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732422103966-3-0] 2024-11-24T04:23:34,402 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: e57b7116ff5e 2024-11-24T04:23:34,409 DEBUG [pool-277-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422212763/_temporary/0/_temporary/attempt_local189592895_0014_r_000000_0/f/2f86776a1bf348cf891942d266bef351 is 59, key is row-test-1732422103966-1-10/f:q1/1732422135476/Put/seqid=0 2024-11-24T04:23:34,411 WARN [Thread-1803 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [e57b7116ff5e:36347] Specified favored nodes: [e57b7116ff5e:36347] 2024-11-24T04:23:34,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741985_1161 (size=11124) 2024-11-24T04:23:34,817 DEBUG [pool-277-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422212763/_temporary/0/_temporary/attempt_local189592895_0014_r_000000_0/mob/efae6e0dbae946a3861a435f57d4ed72 is 192, key is row-test-1732422103966-3-100/mob:q1/1732422135842/Put/seqid=0 2024-11-24T04:23:34,821 WARN [Thread-1805 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [e57b7116ff5e:36347] Specified favored nodes: [e57b7116ff5e:36347] 2024-11-24T04:23:34,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741986_1162 (size=13403) 2024-11-24T04:23:35,263 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe5abc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:35,263 DEBUG [pool-277-thread-1 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:35,263 DEBUG [pool-277-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:35,264 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:35,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:35,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:35,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ac464c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:35,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:35,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:35,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:35,266 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:35,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4715da18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:35,267 DEBUG [pool-277-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:35,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:35,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:35,269 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:35,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:35,271 INFO [pool-277-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-277-thread-1. 2024-11-24T04:23:35,271 DEBUG [pool-277-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:35,271 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:35,272 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:35,272 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:35,272 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row0] 2024-11-24T04:23:35,272 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: e57b7116ff5e 2024-11-24T04:23:35,278 DEBUG [pool-277-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422212763/_temporary/0/_temporary/attempt_local189592895_0014_r_000001_0/f/020ebfc3514b4f67a6a89e7b0d504ef0 is 37, key is row10/f:q1/1732422106681/Put/seqid=0 2024-11-24T04:23:35,280 WARN [Thread-1820 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [e57b7116ff5e:36347] Specified favored nodes: [e57b7116ff5e:36347] 2024-11-24T04:23:35,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741987_1163 (size=8741) 2024-11-24T04:23:36,209 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422212763 2024-11-24T04:23:36,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d9c232, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,209 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:36,210 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:36,210 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:36,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:36,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:36,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73bf233b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:36,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:36,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,211 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:36,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f5b65d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,212 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:36,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:36,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:36,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:36,216 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422212763/_SUCCESS 2024-11-24T04:23:36,223 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:36,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:36,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@18ce4495 2024-11-24T04:23:36,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:36,226 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:36,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:36,232 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T04:23:36,239 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422212763/f/020ebfc3514b4f67a6a89e7b0d504ef0 first=Optional[row0] last=Optional[row98] 2024-11-24T04:23:36,244 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422212763/f/2f86776a1bf348cf891942d266bef351 first=Optional[row-test-1732422103966-1-0] last=Optional[row-test-1732422103966-1-98] 2024-11-24T04:23:36,247 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422212763/mob/efae6e0dbae946a3861a435f57d4ed72 first=Optional[row-test-1732422103966-3-0] last=Optional[row-test-1732422103966-3-99] 2024-11-24T04:23:36,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422212763/mob/efae6e0dbae946a3861a435f57d4ed72 for inclusion in 69e3c311ed1f46d175b4045bb296446a/mob 2024-11-24T04:23:36,251 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422212763/f/020ebfc3514b4f67a6a89e7b0d504ef0 for inclusion in a15495df03a7b8d0d73077c21d437b8b/f 2024-11-24T04:23:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(626): HFile bounds: first=row-test-1732422103966-3-0 last=row-test-1732422103966-3-99 2024-11-24T04:23:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-11-24T04:23:36,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422212763/f/2f86776a1bf348cf891942d266bef351 for inclusion in 69e3c311ed1f46d175b4045bb296446a/f 2024-11-24T04:23:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HStore(626): HFile bounds: first=row0 last=row98 2024-11-24T04:23:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HStore(628): Region bounds: first=row0 last= 2024-11-24T04:23:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HRegion(2603): Flush status journal for a15495df03a7b8d0d73077c21d437b8b: 2024-11-24T04:23:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422212763/f/020ebfc3514b4f67a6a89e7b0d504ef0 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__keg2kdcbm3lheg1nm9bkto4136snhlujf3rm786rrp7idj2741lte6uncvmma8ei/f/020ebfc3514b4f67a6a89e7b0d504ef0 2024-11-24T04:23:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(626): HFile bounds: first=row-test-1732422103966-1-0 last=row-test-1732422103966-1-98 2024-11-24T04:23:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-11-24T04:23:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HRegion(2603): Flush status journal for 69e3c311ed1f46d175b4045bb296446a: 2024-11-24T04:23:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422212763/mob/efae6e0dbae946a3861a435f57d4ed72 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__keg2kdcbm3lheg1nm9bkto4136snhlujf3rm786rrp7idj2741lte6uncvmma8ei/mob/efae6e0dbae946a3861a435f57d4ed72 2024-11-24T04:23:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__keg2kdcbm3lheg1nm9bkto4136snhlujf3rm786rrp7idj2741lte6uncvmma8ei/f/020ebfc3514b4f67a6a89e7b0d504ef0 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a15495df03a7b8d0d73077c21d437b8b/f/101a55171f5f437ab0b5261d8dbbcac4_SeqId_4_ 2024-11-24T04:23:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f159546, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:36,261 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:36,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:36,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__keg2kdcbm3lheg1nm9bkto4136snhlujf3rm786rrp7idj2741lte6uncvmma8ei/mob/efae6e0dbae946a3861a435f57d4ed72 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/mob/a207e7513b01423c8f51ffad50c34cc2_SeqId_4_ 2024-11-24T04:23:36,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8d478a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:36,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422212763/f/2f86776a1bf348cf891942d266bef351 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__keg2kdcbm3lheg1nm9bkto4136snhlujf3rm786rrp7idj2741lte6uncvmma8ei/f/2f86776a1bf348cf891942d266bef351 2024-11-24T04:23:36,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,262 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:36,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0ae433, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__keg2kdcbm3lheg1nm9bkto4136snhlujf3rm786rrp7idj2741lte6uncvmma8ei/f/2f86776a1bf348cf891942d266bef351 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/f/a121baa1f9cd416598aecd3f82e763a5_SeqId_4_ 2024-11-24T04:23:36,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:36,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@62c19fa 2024-11-24T04:23:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159bc28c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:36,265 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:36,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:36,265 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:36,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:36,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c7becf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:36,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:36,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:36,266 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:36,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@659ad306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:36,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:36,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:36,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:36,268 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:36,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@76f222bc 2024-11-24T04:23:36,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:36,269 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:36,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:36,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:36,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:36,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:36,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:36,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:36,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:36,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:36,284 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347. 2024-11-24T04:23:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:36,285 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__keg2kdcbm3lheg1nm9bkto4136snhlujf3rm786rrp7idj2741lte6uncvmma8ei/f/020ebfc3514b4f67a6a89e7b0d504ef0 into a15495df03a7b8d0d73077c21d437b8b/f as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a15495df03a7b8d0d73077c21d437b8b/f/101a55171f5f437ab0b5261d8dbbcac4_SeqId_4_ - updating store file list. 2024-11-24T04:23:36,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:36,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347. 2024-11-24T04:23:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,289 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@234d5427, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:36,291 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:36,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:36,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:36,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ff7abea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:36,291 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a15495df03a7b8d0d73077c21d437b8b/f/101a55171f5f437ab0b5261d8dbbcac4_SeqId_4_ into a15495df03a7b8d0d73077c21d437b8b/f 2024-11-24T04:23:36,291 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__keg2kdcbm3lheg1nm9bkto4136snhlujf3rm786rrp7idj2741lte6uncvmma8ei/f/020ebfc3514b4f67a6a89e7b0d504ef0 into a15495df03a7b8d0d73077c21d437b8b/f (new location: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a15495df03a7b8d0d73077c21d437b8b/f/101a55171f5f437ab0b5261d8dbbcac4_SeqId_4_) 2024-11-24T04:23:36,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:36,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__keg2kdcbm3lheg1nm9bkto4136snhlujf3rm786rrp7idj2741lte6uncvmma8ei/f/020ebfc3514b4f67a6a89e7b0d504ef0 2024-11-24T04:23:36,292 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:36,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae944fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4797d0bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:36,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:36,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b34be1e 2024-11-24T04:23:36,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:36,294 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:36,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:36,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:36,295 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:36,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fdc4e43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:36,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:36,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:36,296 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:36,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8502c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:36,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:36,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:36,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:36,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@469e250f 2024-11-24T04:23:36,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:36,297 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:36,298 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T04:23:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:36,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:36,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:36,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T04:23:36,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:36,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:36,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:36,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:36,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:36,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347. 2024-11-24T04:23:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:36,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__keg2kdcbm3lheg1nm9bkto4136snhlujf3rm786rrp7idj2741lte6uncvmma8ei/f/2f86776a1bf348cf891942d266bef351 into 69e3c311ed1f46d175b4045bb296446a/f as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/f/a121baa1f9cd416598aecd3f82e763a5_SeqId_4_ - updating store file list. 2024-11-24T04:23:36,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:36,320 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347. 2024-11-24T04:23:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:36,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/f/a121baa1f9cd416598aecd3f82e763a5_SeqId_4_ into 69e3c311ed1f46d175b4045bb296446a/f 2024-11-24T04:23:36,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__keg2kdcbm3lheg1nm9bkto4136snhlujf3rm786rrp7idj2741lte6uncvmma8ei/f/2f86776a1bf348cf891942d266bef351 into 69e3c311ed1f46d175b4045bb296446a/f (new location: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/f/a121baa1f9cd416598aecd3f82e763a5_SeqId_4_) 2024-11-24T04:23:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__keg2kdcbm3lheg1nm9bkto4136snhlujf3rm786rrp7idj2741lte6uncvmma8ei/f/2f86776a1bf348cf891942d266bef351 2024-11-24T04:23:36,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__keg2kdcbm3lheg1nm9bkto4136snhlujf3rm786rrp7idj2741lte6uncvmma8ei/mob/efae6e0dbae946a3861a435f57d4ed72 into 69e3c311ed1f46d175b4045bb296446a/mob as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/mob/a207e7513b01423c8f51ffad50c34cc2_SeqId_4_ - updating store file list. 2024-11-24T04:23:36,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/mob/a207e7513b01423c8f51ffad50c34cc2_SeqId_4_ into 69e3c311ed1f46d175b4045bb296446a/mob 2024-11-24T04:23:36,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__keg2kdcbm3lheg1nm9bkto4136snhlujf3rm786rrp7idj2741lte6uncvmma8ei/mob/efae6e0dbae946a3861a435f57d4ed72 into 69e3c311ed1f46d175b4045bb296446a/mob (new location: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/mob/a207e7513b01423c8f51ffad50c34cc2_SeqId_4_) 2024-11-24T04:23:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__keg2kdcbm3lheg1nm9bkto4136snhlujf3rm786rrp7idj2741lte6uncvmma8ei/mob/efae6e0dbae946a3861a435f57d4ed72 2024-11-24T04:23:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@301bd2c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:36,329 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:36,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:36,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:36,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31539b96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:36,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:36,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,331 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:36,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1221f133, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:36,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:36,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@579c0d0a 2024-11-24T04:23:36,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:36,333 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T04:23:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:36,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:36,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:36,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T04:23:36,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:36,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:36,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:36,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347. 2024-11-24T04:23:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:36,350 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:23:36,350 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:36,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:36,350 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-11-24T04:23:36,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 , the old value is region=table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=e57b7116ff5e:36347 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T04:23:36,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T04:23:36,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 from cache 2024-11-24T04:23:36,351 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-24T04:23:36,356 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:36,369 INFO [Time-limited test {}] impl.RestoreTablesClient(181): Restoring 'test-1732422103966' to 'table1' from log dirs: hdfs://localhost:45347/backupUT/backup_1732422186353/default/test-1732422103966/f2/0f50a97b767c46f78559b14c3a21db99 2024-11-24T04:23:36,373 DEBUG [Time-limited test {}] util.RestoreTool(177): Found descriptor 'test-1732422103966', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} through backup_1732422186353 2024-11-24T04:23:36,375 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table table1 from 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:23:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=table1 2024-11-24T04:23:36,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741988_1164 (size=938) 2024-11-24T04:23:36,461 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'table1', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=e57b7116ff5e:36347 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 34 more 2024-11-24T04:23:36,791 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/.tabledesc/.tableinfo.0000000001.938 2024-11-24T04:23:36,791 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/.tabledesc/.tableinfo.0000000002.938 2024-11-24T04:23:36,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=table1}] 2024-11-24T04:23:36,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=69e3c311ed1f46d175b4045bb296446a, REOPEN/MOVE}, {pid=83, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=a15495df03a7b8d0d73077c21d437b8b, REOPEN/MOVE}] 2024-11-24T04:23:36,802 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=69e3c311ed1f46d175b4045bb296446a, REOPEN/MOVE 2024-11-24T04:23:36,802 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=83, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=a15495df03a7b8d0d73077c21d437b8b, REOPEN/MOVE 2024-11-24T04:23:36,803 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=69e3c311ed1f46d175b4045bb296446a, regionState=CLOSING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:36,803 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=a15495df03a7b8d0d73077c21d437b8b, regionState=CLOSING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:36,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=69e3c311ed1f46d175b4045bb296446a, REOPEN/MOVE because future has completed 2024-11-24T04:23:36,805 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T04:23:36,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 69e3c311ed1f46d175b4045bb296446a, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:23:36,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=a15495df03a7b8d0d73077c21d437b8b, REOPEN/MOVE because future has completed 2024-11-24T04:23:36,806 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T04:23:36,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=83, state=RUNNABLE, hasLock=false; CloseRegionProcedure a15495df03a7b8d0d73077c21d437b8b, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:23:36,959 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] handler.UnassignRegionHandler(122): Close 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:36,959 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T04:23:36,959 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1722): Closing 69e3c311ed1f46d175b4045bb296446a, disabling compactions & flushes 2024-11-24T04:23:36,959 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1755): Closing region table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:36,960 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:36,960 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1843): Acquired close lock on table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. after waiting 0 ms 2024-11-24T04:23:36,960 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1853): Updates disabled for region table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:36,968 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-24T04:23:36,969 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:23:36,969 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1973): Closed table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:36,969 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1676): Region close journal for 69e3c311ed1f46d175b4045bb296446a: Waiting for close lock at 1732422216959Running coprocessor pre-close hooks at 1732422216959Disabling compacts and flushes for region at 1732422216959Disabling writes for close at 1732422216960 (+1 ms)Writing region close event to WAL at 1732422216962 (+2 ms)Running coprocessor post-close hooks at 1732422216969 (+7 ms)Closed at 1732422216969 2024-11-24T04:23:36,970 WARN [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegionServer(3299): Not adding moved region record: 69e3c311ed1f46d175b4045bb296446a to self. 2024-11-24T04:23:36,972 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] handler.UnassignRegionHandler(157): Closed 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:36,973 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:36,973 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T04:23:36,973 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing a15495df03a7b8d0d73077c21d437b8b, disabling compactions & flushes 2024-11-24T04:23:36,973 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:36,973 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:36,973 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. after waiting 0 ms 2024-11-24T04:23:36,973 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:36,974 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=69e3c311ed1f46d175b4045bb296446a, regionState=CLOSED 2024-11-24T04:23:36,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 69e3c311ed1f46d175b4045bb296446a, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:23:36,979 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a15495df03a7b8d0d73077c21d437b8b/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-24T04:23:36,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=82 2024-11-24T04:23:36,980 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:23:36,980 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:36,980 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for a15495df03a7b8d0d73077c21d437b8b: Waiting for close lock at 1732422216973Running coprocessor pre-close hooks at 1732422216973Disabling compacts and flushes for region at 1732422216973Disabling writes for close at 1732422216973Writing region close event to WAL at 1732422216974 (+1 ms)Running coprocessor post-close hooks at 1732422216980 (+6 ms)Closed at 1732422216980 2024-11-24T04:23:36,980 WARN [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: a15495df03a7b8d0d73077c21d437b8b to self. 2024-11-24T04:23:36,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 69e3c311ed1f46d175b4045bb296446a, server=e57b7116ff5e,36347,1732422095008 in 172 msec 2024-11-24T04:23:36,981 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=69e3c311ed1f46d175b4045bb296446a, REOPEN/MOVE; state=CLOSED, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=true 2024-11-24T04:23:36,981 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:36,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=a15495df03a7b8d0d73077c21d437b8b, regionState=CLOSED 2024-11-24T04:23:36,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=83, state=RUNNABLE, hasLock=false; CloseRegionProcedure a15495df03a7b8d0d73077c21d437b8b, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:23:36,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=83 2024-11-24T04:23:36,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=83, state=SUCCESS, hasLock=false; CloseRegionProcedure a15495df03a7b8d0d73077c21d437b8b, server=e57b7116ff5e,36347,1732422095008 in 178 msec 2024-11-24T04:23:36,986 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=83, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=a15495df03a7b8d0d73077c21d437b8b, REOPEN/MOVE; state=CLOSED, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=true 2024-11-24T04:23:37,132 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=a15495df03a7b8d0d73077c21d437b8b, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:37,132 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=69e3c311ed1f46d175b4045bb296446a, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:37,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=69e3c311ed1f46d175b4045bb296446a, REOPEN/MOVE because future has completed 2024-11-24T04:23:37,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69e3c311ed1f46d175b4045bb296446a, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:23:37,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=a15495df03a7b8d0d73077c21d437b8b, REOPEN/MOVE because future has completed 2024-11-24T04:23:37,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=83, state=RUNNABLE, hasLock=false; OpenRegionProcedure a15495df03a7b8d0d73077c21d437b8b, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:23:37,294 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:37,295 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => a15495df03a7b8d0d73077c21d437b8b, NAME => 'table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b.', STARTKEY => 'row0', ENDKEY => ''} 2024-11-24T04:23:37,295 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:23:37,295 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:37,295 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:23:37,296 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:37,296 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:37,298 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:37,300 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a15495df03a7b8d0d73077c21d437b8b columnFamilyName f 2024-11-24T04:23:37,300 DEBUG [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:37,307 DEBUG [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a15495df03a7b8d0d73077c21d437b8b/f/101a55171f5f437ab0b5261d8dbbcac4_SeqId_4_ 2024-11-24T04:23:37,307 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] regionserver.HStore(327): Store=a15495df03a7b8d0d73077c21d437b8b/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:37,308 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:37,309 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a15495df03a7b8d0d73077c21d437b8b columnFamilyName f2 2024-11-24T04:23:37,309 DEBUG [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:37,310 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] regionserver.HStore(327): Store=a15495df03a7b8d0d73077c21d437b8b/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:37,310 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:37,311 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a15495df03a7b8d0d73077c21d437b8b columnFamilyName mob 2024-11-24T04:23:37,312 DEBUG [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:37,312 INFO [StoreOpener-a15495df03a7b8d0d73077c21d437b8b-1 {}] regionserver.HStore(327): Store=a15495df03a7b8d0d73077c21d437b8b/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:37,312 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:37,313 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:37,314 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-24T04:23:37,314 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:37,315 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:37,315 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:37,316 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:23:37,317 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:37,317 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened a15495df03a7b8d0d73077c21d437b8b; next sequenceid=7; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9551843200, jitterRate=-0.11041527986526489}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:23:37,317 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:37,318 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for a15495df03a7b8d0d73077c21d437b8b: Running coprocessor pre-open hook at 1732422217296Writing region info on filesystem at 1732422217296Initializing all the Stores at 1732422217298 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422217298Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422217298Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422217298Cleaning up temporary data from old regions at 1732422217315 (+17 ms)Running coprocessor post-open hooks at 1732422217317 (+2 ms)Region opened successfully at 1732422217318 (+1 ms) 2024-11-24T04:23:37,319 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b., pid=87, masterSystemTime=1732422217291 2024-11-24T04:23:37,320 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:37,321 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:37,321 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(132): Open table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:37,321 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7752): Opening region: {ENCODED => 69e3c311ed1f46d175b4045bb296446a, NAME => 'table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a.', STARTKEY => '', ENDKEY => 'row0'} 2024-11-24T04:23:37,321 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=a15495df03a7b8d0d73077c21d437b8b, regionState=OPEN, openSeqNum=7, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:37,321 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:23:37,322 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:37,322 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(898): Instantiated table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:23:37,322 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7794): checking encryption for 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:37,322 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7797): checking classloading for 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:37,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=83, state=RUNNABLE, hasLock=false; OpenRegionProcedure a15495df03a7b8d0d73077c21d437b8b, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:23:37,323 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:37,324 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69e3c311ed1f46d175b4045bb296446a columnFamilyName f 2024-11-24T04:23:37,324 DEBUG [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:37,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=83 2024-11-24T04:23:37,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=83, state=SUCCESS, hasLock=false; OpenRegionProcedure a15495df03a7b8d0d73077c21d437b8b, server=e57b7116ff5e,36347,1732422095008 in 184 msec 2024-11-24T04:23:37,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=a15495df03a7b8d0d73077c21d437b8b, REOPEN/MOVE in 525 msec 2024-11-24T04:23:37,331 DEBUG [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/f/a121baa1f9cd416598aecd3f82e763a5_SeqId_4_ 2024-11-24T04:23:37,331 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] regionserver.HStore(327): Store=69e3c311ed1f46d175b4045bb296446a/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:37,331 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:37,332 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69e3c311ed1f46d175b4045bb296446a columnFamilyName f2 2024-11-24T04:23:37,333 DEBUG [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:37,333 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] regionserver.HStore(327): Store=69e3c311ed1f46d175b4045bb296446a/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:37,333 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:37,334 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69e3c311ed1f46d175b4045bb296446a columnFamilyName mob 2024-11-24T04:23:37,335 DEBUG [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:37,340 DEBUG [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/mob/a207e7513b01423c8f51ffad50c34cc2_SeqId_4_ 2024-11-24T04:23:37,340 INFO [StoreOpener-69e3c311ed1f46d175b4045bb296446a-1 {}] regionserver.HStore(327): Store=69e3c311ed1f46d175b4045bb296446a/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:37,341 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1038): replaying wal for 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:37,341 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:37,342 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:37,343 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1048): stopping wal replay for 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:37,343 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1060): Cleaning up temporary data for 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:37,344 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:23:37,345 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1093): writing seq id for 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:37,345 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1114): Opened 69e3c311ed1f46d175b4045bb296446a; next sequenceid=7; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10612568480, jitterRate=-0.011627539992332458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:23:37,346 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:37,346 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1006): Region open journal for 69e3c311ed1f46d175b4045bb296446a: Running coprocessor pre-open hook at 1732422217322Writing region info on filesystem at 1732422217322Initializing all the Stores at 1732422217323 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422217323Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422217323Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422217323Cleaning up temporary data from old regions at 1732422217343 (+20 ms)Running coprocessor post-open hooks at 1732422217346 (+3 ms)Region opened successfully at 1732422217346 2024-11-24T04:23:37,346 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., pid=86, masterSystemTime=1732422217291 2024-11-24T04:23:37,348 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:37,348 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(153): Opened table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:37,349 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=69e3c311ed1f46d175b4045bb296446a, regionState=OPEN, openSeqNum=7, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:37,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69e3c311ed1f46d175b4045bb296446a, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:23:37,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=82 2024-11-24T04:23:37,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure 69e3c311ed1f46d175b4045bb296446a, server=e57b7116ff5e,36347,1732422095008 in 214 msec 2024-11-24T04:23:37,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-24T04:23:37,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=69e3c311ed1f46d175b4045bb296446a, REOPEN/MOVE in 551 msec 2024-11-24T04:23:37,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-24T04:23:37,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=table1 in 558 msec 2024-11-24T04:23:37,358 DEBUG [PEWorker-2 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=table1 2024-11-24T04:23:37,358 DEBUG [PEWorker-2 {}] backup.HFileArchiver(265): No files to dispose of in table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., family=f3 2024-11-24T04:23:37,359 DEBUG [PEWorker-2 {}] backup.HFileArchiver(265): No files to dispose of in table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b., family=f3 2024-11-24T04:23:37,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; ModifyTableProcedure table=table1 in 985 msec 2024-11-24T04:23:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T04:23:37,362 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:table1 completed 2024-11-24T04:23:37,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T04:23:37,366 INFO [Time-limited test {}] util.RestoreTool(200): Changed table1 to: 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:23:37,366 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore incremental backup from directory hdfs://localhost:45347/backupUT/backup_1732422186353/default/test-1732422103966/f2/0f50a97b767c46f78559b14c3a21db99 from hbase tables test-1732422103966 to tables table1 2024-11-24T04:23:37,366 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore test-1732422103966 into table1 2024-11-24T04:23:37,368 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422217367 from hdfs://localhost:45347/backupUT/backup_1732422186353/default/test-1732422103966/f2/0f50a97b767c46f78559b14c3a21db99 2024-11-24T04:23:37,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e26fa40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:37,369 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:37,369 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:37,369 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:37,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:37,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:37,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b62fb87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:37,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:37,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:37,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:37,370 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:37,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f714d5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:37,371 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:37,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:37,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@270ae8ed 2024-11-24T04:23:37,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:37,373 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:37,373 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-24T04:23:37,374 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table table1 2024-11-24T04:23:37,374 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T04:23:37,374 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:37,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:37,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:37,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:37,377 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: 2024-11-24T04:23:37,377 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: row0 2024-11-24T04:23:37,377 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-24T04:23:37,378 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/partitions_7b8452ab-9dfc-4625-b195-65e3148b8491 2024-11-24T04:23:37,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741989_1165 (size=177) 2024-11-24T04:23:37,784 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:37,785 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:37,785 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,082 WARN [regionserver/e57b7116ff5e:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 16, running: 1 2024-11-24T04:23:38,083 DEBUG [regionserver/e57b7116ff5e:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-24T04:23:38,648 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-10847591765467863786.jar 2024-11-24T04:23:38,648 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,648 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,648 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,648 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,649 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,649 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,649 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,649 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,649 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,649 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:23:38,650 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T04:23:38,650 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T04:23:38,650 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T04:23:38,650 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T04:23:38,650 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T04:23:38,650 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T04:23:38,650 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T04:23:38,651 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T04:23:38,651 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T04:23:38,651 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T04:23:38,651 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,651 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,652 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,652 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,652 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,652 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:38,652 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:23:38,652 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: table1 2024-11-24T04:23:38,652 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:23:38,652 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.incrementalRestoreTable(RestoreTool.java:205) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:185) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:38,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:38,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:38,653 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:38,653 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-11-24T04:23:38,653 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:23:38,662 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:23:38,672 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-24T04:23:38,751 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local523780817_0015_eeaac2c8-43d9-4016-99a6-a7838d2c5f93/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-24T04:23:38,751 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local523780817_0015_eeaac2c8-43d9-4016-99a6-a7838d2c5f93/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-24T04:23:38,811 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45347/backupUT/backup_1732422186353/default/test-1732422103966/f2/0f50a97b767c46f78559b14c3a21db99 2024-11-24T04:23:38,813 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T04:23:38,844 DEBUG [pool-281-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7750bea7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:38,844 DEBUG [pool-281-thread-1 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:38,844 DEBUG [pool-281-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:38,845 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:38,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:38,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:38,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e834737, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:38,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:38,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:38,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:38,846 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:38,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@436f2520, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:38,847 DEBUG [pool-281-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:38,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:38,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:38,849 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:38,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1732422103966-2-0', locateType=CURRENT is [region=table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., hostname=e57b7116ff5e,36347,1732422095008, seqNum=7] 2024-11-24T04:23:38,851 INFO [pool-281-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-281-thread-1. 2024-11-24T04:23:38,851 DEBUG [pool-281-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:38,851 DEBUG [pool-281-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:38,851 DEBUG [pool-281-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:38,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:38,851 DEBUG [pool-281-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732422103966-2-0] 2024-11-24T04:23:38,851 DEBUG [pool-281-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: e57b7116ff5e 2024-11-24T04:23:38,854 DEBUG [pool-281-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422217367/_temporary/0/_temporary/attempt_local523780817_0015_r_000000_0/f2/ae393885f1f34a58bcb254750a099c06 is 58, key is row-test-1732422103966-2-0/f2:q1/1732422181343/Put/seqid=0 2024-11-24T04:23:38,856 WARN [Thread-1884 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [e57b7116ff5e:36347] Specified favored nodes: [e57b7116ff5e:36347] 2024-11-24T04:23:38,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741990_1166 (size=5608) 2024-11-24T04:23:39,797 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422217367 2024-11-24T04:23:39,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@106bbbf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:39,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:39,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:39,800 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:39,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:39,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:39,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46c574a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:39,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:39,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:39,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:39,801 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:39,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29427cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:39,802 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:39,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:39,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:39,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:39,807 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422217367/_SUCCESS 2024-11-24T04:23:39,810 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:39,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:39,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6192ebad 2024-11-24T04:23:39,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:39,812 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:39,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., hostname=e57b7116ff5e,36347,1732422095008, seqNum=7] 2024-11-24T04:23:39,817 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T04:23:39,822 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422217367/f2/ae393885f1f34a58bcb254750a099c06 first=Optional[row-test-1732422103966-2-0] last=Optional[row-test-1732422103966-2-6] 2024-11-24T04:23:39,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422217367/f2/ae393885f1f34a58bcb254750a099c06 for inclusion in 69e3c311ed1f46d175b4045bb296446a/f2 2024-11-24T04:23:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(626): HFile bounds: first=row-test-1732422103966-2-0 last=row-test-1732422103966-2-6 2024-11-24T04:23:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-11-24T04:23:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HRegion(2603): Flush status journal for 69e3c311ed1f46d175b4045bb296446a: 2024-11-24T04:23:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-default-table1-1732422217367/f2/ae393885f1f34a58bcb254750a099c06 to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__1ph1188e9deg2fs8jobno0e8ackk5len7k0diimths33dqi3ucupd6ft7554cghu/f2/ae393885f1f34a58bcb254750a099c06 2024-11-24T04:23:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__1ph1188e9deg2fs8jobno0e8ackk5len7k0diimths33dqi3ucupd6ft7554cghu/f2/ae393885f1f34a58bcb254750a099c06 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/f2/ca4d4c2f4a114ea5a9bd4a1d62fdea50_SeqId_9_ 2024-11-24T04:23:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b0bb2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:39,834 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:39,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:39,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:39,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47489ec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:39,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:39,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:39,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:39,835 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:39,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cafab07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:39,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:39,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@121d1a20 2024-11-24T04:23:39,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:39,837 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:39,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:39,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:39,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:39,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:39,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:39,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:39,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347. 2024-11-24T04:23:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:39,855 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:39,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__1ph1188e9deg2fs8jobno0e8ackk5len7k0diimths33dqi3ucupd6ft7554cghu/f2/ae393885f1f34a58bcb254750a099c06 into 69e3c311ed1f46d175b4045bb296446a/f2 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/f2/ca4d4c2f4a114ea5a9bd4a1d62fdea50_SeqId_9_ - updating store file list. 2024-11-24T04:23:39,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/f2/ca4d4c2f4a114ea5a9bd4a1d62fdea50_SeqId_9_ into 69e3c311ed1f46d175b4045bb296446a/f2 2024-11-24T04:23:39,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__1ph1188e9deg2fs8jobno0e8ackk5len7k0diimths33dqi3ucupd6ft7554cghu/f2/ae393885f1f34a58bcb254750a099c06 into 69e3c311ed1f46d175b4045bb296446a/f2 (new location: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/f2/ca4d4c2f4a114ea5a9bd4a1d62fdea50_SeqId_9_) 2024-11-24T04:23:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__table1__1ph1188e9deg2fs8jobno0e8ackk5len7k0diimths33dqi3ucupd6ft7554cghu/f2/ae393885f1f34a58bcb254750a099c06 2024-11-24T04:23:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ce15fbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:39,863 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:39,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:39,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:39,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa45ebd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:39,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:39,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:39,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:39,864 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:39,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13426ed5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:39,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:39,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6b31960d 2024-11-24T04:23:39,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:39,866 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T04:23:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:39,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:39,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:39,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T04:23:39,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:39,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:39,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:39,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347. 2024-11-24T04:23:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:39,882 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:39,883 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:23:39,883 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.incrementalRestoreTable(RestoreTool.java:205) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:185) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:39,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:39,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:39,883 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:39,883 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-11-24T04:23:39,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., hostname=e57b7116ff5e,36347,1732422095008, seqNum=7 , the old value is region=table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., hostname=e57b7116ff5e,36347,1732422095008, seqNum=7, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=e57b7116ff5e:36347 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T04:23:39,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., hostname=e57b7116ff5e,36347,1732422095008, seqNum=7 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T04:23:39,884 INFO [Time-limited test {}] impl.RestoreTablesClient(187): test-1732422103966 has been successfully restored to table1 2024-11-24T04:23:39,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., hostname=e57b7116ff5e,36347,1732422095008, seqNum=7 from cache 2024-11-24T04:23:39,884 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-11-24T04:23:39,884 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1732422186353 hdfs://localhost:45347/backupUT/backup_1732422186353/default/test-1732422103966/ 2024-11-24T04:23:39,884 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45347/backupUT/backup_1732422151095/.backup.manifest 2024-11-24T04:23:39,884 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-24T04:23:39,886 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732422151095/.backup.manifest 2024-11-24T04:23:39,886 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'ns2:test-17324221039661' to 'ns2:table2' from full backup image hdfs://localhost:45347/backupUT/backup_1732422151095/ns2/test-17324221039661 2024-11-24T04:23:39,888 DEBUG [Time-limited test {}] util.RestoreTool(295): Retrieved descriptor: 'ns2:test-17324221039661', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} thru backup_1732422186353 2024-11-24T04:23:39,891 INFO [Time-limited test {}] util.RestoreTool(480): Truncating exising target table 'ns2:table2', preserving region splits 2024-11-24T04:23:39,891 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable ns2:table2 2024-11-24T04:23:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=ns2:table2 2024-11-24T04:23:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T04:23:39,894 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422219894"}]},"ts":"1732422219894"} 2024-11-24T04:23:39,896 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2024-11-24T04:23:39,896 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set ns2:table2 to state=DISABLING 2024-11-24T04:23:39,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=ns2:table2}] 2024-11-24T04:23:39,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=589b5c4b38f159789612c7e3246e2779, UNASSIGN}] 2024-11-24T04:23:39,899 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=589b5c4b38f159789612c7e3246e2779, UNASSIGN 2024-11-24T04:23:39,899 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=589b5c4b38f159789612c7e3246e2779, regionState=CLOSING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:39,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=589b5c4b38f159789612c7e3246e2779, UNASSIGN because future has completed 2024-11-24T04:23:39,901 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T04:23:39,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 589b5c4b38f159789612c7e3246e2779, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:23:39,990 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'table1', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=e57b7116ff5e:36347 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 34 more 2024-11-24T04:23:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T04:23:40,054 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 589b5c4b38f159789612c7e3246e2779 2024-11-24T04:23:40,054 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T04:23:40,055 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 589b5c4b38f159789612c7e3246e2779, disabling compactions & flushes 2024-11-24T04:23:40,055 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. 2024-11-24T04:23:40,055 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. 2024-11-24T04:23:40,055 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. after waiting 0 ms 2024-11-24T04:23:40,055 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. 2024-11-24T04:23:40,066 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/589b5c4b38f159789612c7e3246e2779/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-24T04:23:40,068 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:23:40,068 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. 2024-11-24T04:23:40,068 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 589b5c4b38f159789612c7e3246e2779: Waiting for close lock at 1732422220055Running coprocessor pre-close hooks at 1732422220055Disabling compacts and flushes for region at 1732422220055Disabling writes for close at 1732422220055Writing region close event to WAL at 1732422220057 (+2 ms)Running coprocessor post-close hooks at 1732422220068 (+11 ms)Closed at 1732422220068 2024-11-24T04:23:40,070 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 589b5c4b38f159789612c7e3246e2779 2024-11-24T04:23:40,070 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=589b5c4b38f159789612c7e3246e2779, regionState=CLOSED 2024-11-24T04:23:40,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 589b5c4b38f159789612c7e3246e2779, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:23:40,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-24T04:23:40,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 589b5c4b38f159789612c7e3246e2779, server=e57b7116ff5e,36347,1732422095008 in 171 msec 2024-11-24T04:23:40,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-24T04:23:40,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=589b5c4b38f159789612c7e3246e2779, UNASSIGN in 176 msec 2024-11-24T04:23:40,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-24T04:23:40,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=ns2:table2 in 180 msec 2024-11-24T04:23:40,078 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422220078"}]},"ts":"1732422220078"} 2024-11-24T04:23:40,080 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2024-11-24T04:23:40,080 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set ns2:table2 to state=DISABLED 2024-11-24T04:23:40,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=ns2:table2 in 189 msec 2024-11-24T04:23:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T04:23:40,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: ns2:table2 completed 2024-11-24T04:23:40,213 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] master.HMaster$6(2602): Client=jenkins//172.17.0.3 truncate ns2:table2 2024-11-24T04:23:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION, hasLock=false; TruncateTableProcedure (table=ns2:table2 preserveSplits=true) 2024-11-24T04:23:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-24T04:23:40,217 DEBUG [PEWorker-4 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:table2' regions in transition 2024-11-24T04:23:40,222 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/589b5c4b38f159789612c7e3246e2779 2024-11-24T04:23:40,225 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/589b5c4b38f159789612c7e3246e2779/f, FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/589b5c4b38f159789612c7e3246e2779/recovered.edits] 2024-11-24T04:23:40,233 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/589b5c4b38f159789612c7e3246e2779/f/e8a8d2160ae7448882738d7c31ecdf11_SeqId_4_ to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/archive/data/ns2/table2/589b5c4b38f159789612c7e3246e2779/f/e8a8d2160ae7448882738d7c31ecdf11_SeqId_4_ 2024-11-24T04:23:40,238 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/589b5c4b38f159789612c7e3246e2779/recovered.edits/6.seqid to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/archive/data/ns2/table2/589b5c4b38f159789612c7e3246e2779/recovered.edits/6.seqid 2024-11-24T04:23:40,239 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/589b5c4b38f159789612c7e3246e2779 2024-11-24T04:23:40,239 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived ns2:table2 regions 2024-11-24T04:23:40,245 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2024-11-24T04:23:40,247 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'ns2:table2' descriptor. 2024-11-24T04:23:40,247 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'ns2:table2' from region states. 2024-11-24T04:23:40,248 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732422220247"}]},"ts":"9223372036854775807"} 2024-11-24T04:23:40,249 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T04:23:40,249 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 589b5c4b38f159789612c7e3246e2779, NAME => 'ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T04:23:40,250 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'ns2:table2' as deleted. 2024-11-24T04:23:40,250 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732422220250"}]},"ts":"9223372036854775807"} 2024-11-24T04:23:40,251 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table ns2:table2 state from META 2024-11-24T04:23:40,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741991_1167 (size=351) 2024-11-24T04:23:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-24T04:23:40,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-24T04:23:40,664 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8133bf0180eb47986f00610092ee73e6, NAME => 'ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd 2024-11-24T04:23:40,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741992_1168 (size=37) 2024-11-24T04:23:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-24T04:23:41,075 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:23:41,076 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1722): Closing 8133bf0180eb47986f00610092ee73e6, disabling compactions & flushes 2024-11-24T04:23:41,076 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6. 2024-11-24T04:23:41,076 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6. 2024-11-24T04:23:41,076 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6. after waiting 0 ms 2024-11-24T04:23:41,076 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6. 2024-11-24T04:23:41,076 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1973): Closed ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6. 2024-11-24T04:23:41,076 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8133bf0180eb47986f00610092ee73e6: Waiting for close lock at 1732422221076Disabling compacts and flushes for region at 1732422221076Disabling writes for close at 1732422221076Writing region close event to WAL at 1732422221076Closed at 1732422221076 2024-11-24T04:23:41,080 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1732422221079"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422221079"}]},"ts":"1732422221079"} 2024-11-24T04:23:41,084 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T04:23:41,085 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422221085"}]},"ts":"1732422221085"} 2024-11-24T04:23:41,087 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2024-11-24T04:23:41,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=8133bf0180eb47986f00610092ee73e6, ASSIGN}] 2024-11-24T04:23:41,089 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=93, ppid=92, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=8133bf0180eb47986f00610092ee73e6, ASSIGN 2024-11-24T04:23:41,090 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=93, ppid=92, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=8133bf0180eb47986f00610092ee73e6, ASSIGN; state=OFFLINE, location=e57b7116ff5e,36347,1732422095008; forceNewPlan=false, retain=false 2024-11-24T04:23:41,241 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=93 updating hbase:meta row=8133bf0180eb47986f00610092ee73e6, regionState=OPENING, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:41,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=8133bf0180eb47986f00610092ee73e6, ASSIGN because future has completed 2024-11-24T04:23:41,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8133bf0180eb47986f00610092ee73e6, server=e57b7116ff5e,36347,1732422095008}] 2024-11-24T04:23:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-24T04:23:41,409 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6. 2024-11-24T04:23:41,410 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 8133bf0180eb47986f00610092ee73e6, NAME => 'ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:23:41,411 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T04:23:41,411 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 8133bf0180eb47986f00610092ee73e6 2024-11-24T04:23:41,411 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:23:41,411 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 8133bf0180eb47986f00610092ee73e6 2024-11-24T04:23:41,411 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 8133bf0180eb47986f00610092ee73e6 2024-11-24T04:23:41,413 INFO [StoreOpener-8133bf0180eb47986f00610092ee73e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 8133bf0180eb47986f00610092ee73e6 2024-11-24T04:23:41,415 INFO [StoreOpener-8133bf0180eb47986f00610092ee73e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8133bf0180eb47986f00610092ee73e6 columnFamilyName f 2024-11-24T04:23:41,415 DEBUG [StoreOpener-8133bf0180eb47986f00610092ee73e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:41,416 INFO [StoreOpener-8133bf0180eb47986f00610092ee73e6-1 {}] regionserver.HStore(327): Store=8133bf0180eb47986f00610092ee73e6/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:23:41,416 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 8133bf0180eb47986f00610092ee73e6 2024-11-24T04:23:41,417 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/8133bf0180eb47986f00610092ee73e6 2024-11-24T04:23:41,417 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/8133bf0180eb47986f00610092ee73e6 2024-11-24T04:23:41,418 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 8133bf0180eb47986f00610092ee73e6 2024-11-24T04:23:41,418 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 8133bf0180eb47986f00610092ee73e6 2024-11-24T04:23:41,420 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 8133bf0180eb47986f00610092ee73e6 2024-11-24T04:23:41,423 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/8133bf0180eb47986f00610092ee73e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:23:41,423 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 8133bf0180eb47986f00610092ee73e6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10397305280, jitterRate=-0.031675487756729126}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:23:41,424 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8133bf0180eb47986f00610092ee73e6 2024-11-24T04:23:41,425 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 8133bf0180eb47986f00610092ee73e6: Running coprocessor pre-open hook at 1732422221412Writing region info on filesystem at 1732422221412Initializing all the Stores at 1732422221413 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422221413Cleaning up temporary data from old regions at 1732422221418 (+5 ms)Running coprocessor post-open hooks at 1732422221424 (+6 ms)Region opened successfully at 1732422221425 (+1 ms) 2024-11-24T04:23:41,426 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6., pid=94, masterSystemTime=1732422221401 2024-11-24T04:23:41,428 DEBUG [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6. 2024-11-24T04:23:41,428 INFO [RS_OPEN_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6. 2024-11-24T04:23:41,429 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=93 updating hbase:meta row=8133bf0180eb47986f00610092ee73e6, regionState=OPEN, openSeqNum=2, regionLocation=e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:41,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8133bf0180eb47986f00610092ee73e6, server=e57b7116ff5e,36347,1732422095008 because future has completed 2024-11-24T04:23:41,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-24T04:23:41,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; OpenRegionProcedure 8133bf0180eb47986f00610092ee73e6, server=e57b7116ff5e,36347,1732422095008 in 185 msec 2024-11-24T04:23:41,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-24T04:23:41,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=8133bf0180eb47986f00610092ee73e6, ASSIGN in 345 msec 2024-11-24T04:23:41,436 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422221435"}]},"ts":"1732422221435"} 2024-11-24T04:23:41,437 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2024-11-24T04:23:41,437 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:table2' completed 2024-11-24T04:23:41,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; TruncateTableProcedure (table=ns2:table2 preserveSplits=true) in 1.2240 sec 2024-11-24T04:23:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-24T04:23:42,362 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: TRUNCATE, Table Name: ns2:table2 completed 2024-11-24T04:23:42,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-24T04:23:42,373 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45347/backupUT/backup_1732422151095/ns2/test-17324221039661/archive/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1 from hbase tables ns2:test-17324221039661 to tables ns2:table2 2024-11-24T04:23:42,373 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-17324221039661 into ns2:table2 2024-11-24T04:23:42,377 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-ns2-table2-1732422222373 from hdfs://localhost:45347/backupUT/backup_1732422151095/ns2/test-17324221039661/archive/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1 2024-11-24T04:23:42,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@262dca66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:42,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:42,378 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:42,378 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:42,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:42,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:42,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4abac822, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:42,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:42,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:42,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:42,380 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:42,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41f5e747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:42,381 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:42,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:42,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1b22b2e4 2024-11-24T04:23:42,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:42,383 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:42,384 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-24T04:23:42,384 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:table2 2024-11-24T04:23:42,384 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-24T04:23:42,384 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:42,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:42,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:42,387 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:42,388 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:table2: 2024-11-24T04:23:42,388 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 1 reduce partitions to match current region count for all tables 2024-11-24T04:23:42,389 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/partitions_32b213e1-095c-43af-a72d-494f7b154509 2024-11-24T04:23:42,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741993_1169 (size=153) 2024-11-24T04:23:42,474 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-24T04:23:42,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:42,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:42,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,662 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop-16796369228232624652.jar 2024-11-24T04:23:43,662 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,662 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,663 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,663 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,663 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,663 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,663 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,664 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,664 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,664 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:23:43,664 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T04:23:43,664 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T04:23:43,664 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T04:23:43,665 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T04:23:43,665 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T04:23:43,665 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T04:23:43,665 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T04:23:43,665 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T04:23:43,665 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T04:23:43,666 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T04:23:43,666 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,666 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,666 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,666 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,666 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,667 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-24T04:23:43,667 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T04:23:43,667 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:table2 2024-11-24T04:23:43,667 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:23:43,667 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:43,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:43,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:43,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:43,667 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-11-24T04:23:43,668 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T04:23:43,676 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T04:23:43,686 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-24T04:23:43,776 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local262579560_0016_db073f4e-3078-483d-8ce6-cbf833c2e99f/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-24T04:23:43,776 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/mapreduce.cluster.local.dir/job_local262579560_0016_db073f4e-3078-483d-8ce6-cbf833c2e99f/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-24T04:23:43,817 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45347/backupUT/backup_1732422151095/ns2/test-17324221039661/archive/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/933ab35b68994dc8be3ccc706f0c0aa9 2024-11-24T04:23:43,820 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T04:23:43,850 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45347/backupUT/backup_1732422151095/ns2/test-17324221039661/archive/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/f/1d220a07c36347908f109231fa9cf854 2024-11-24T04:23:43,853 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T04:23:43,878 DEBUG [pool-286-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@409a2c6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:43,878 DEBUG [pool-286-thread-1 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:43,878 DEBUG [pool-286-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:43,879 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:43,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:43,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:43,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5926fa16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:43,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:43,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:43,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:43,880 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:43,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c2b9aa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:43,881 DEBUG [pool-286-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:43,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:43,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:43,883 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:43,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='row-t20', locateType=CURRENT is [region=ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:43,884 INFO [pool-286-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-286-thread-1. 2024-11-24T04:23:43,885 DEBUG [pool-286-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:43,885 DEBUG [pool-286-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:43,885 DEBUG [pool-286-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:43,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:43,885 DEBUG [pool-286-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-11-24T04:23:43,885 DEBUG [pool-286-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: e57b7116ff5e 2024-11-24T04:23:43,889 DEBUG [pool-286-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-ns2-table2-1732422222373/_temporary/0/_temporary/attempt_local262579560_0016_r_000000_0/f/8f813cb3d88c42c28ab26a624527777a is 38, key is row-t20/f:q1/1732422135882/Put/seqid=0 2024-11-24T04:23:43,891 WARN [Thread-1959 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [e57b7116ff5e:36347] Specified favored nodes: [e57b7116ff5e:36347] 2024-11-24T04:23:43,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741994_1170 (size=8934) 2024-11-24T04:23:44,817 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-ns2-table2-1732422222373 2024-11-24T04:23:44,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3034eb1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:44,819 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:44,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:44,820 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:44,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:44,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:44,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@508b461e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:44,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:44,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:44,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:44,822 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:44,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26ac0760, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:44,822 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:44,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:44,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:44,824 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:44,826 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-ns2-table2-1732422222373/_SUCCESS 2024-11-24T04:23:44,829 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:44,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:44,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@76b77684 2024-11-24T04:23:44,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:44,831 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:44,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:44,836 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-24T04:23:44,841 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-ns2-table2-1732422222373/f/8f813cb3d88c42c28ab26a624527777a first=Optional[row-t20] last=Optional[row98] 2024-11-24T04:23:44,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-ns2-table2-1732422222373/f/8f813cb3d88c42c28ab26a624527777a for inclusion in 8133bf0180eb47986f00610092ee73e6/f 2024-11-24T04:23:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(626): HFile bounds: first=row-t20 last=row98 2024-11-24T04:23:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T04:23:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HRegion(2603): Flush status journal for 8133bf0180eb47986f00610092ee73e6: 2024-11-24T04:23:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/hbase-staging/bulk_output-ns2-table2-1732422222373/f/8f813cb3d88c42c28ab26a624527777a to hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__ns2_table2__3uhm2ip8fgvd6t71aamlili27djecra7poo593emtkqovgn31ej91mbv5rijrcrv/f/8f813cb3d88c42c28ab26a624527777a 2024-11-24T04:23:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__ns2_table2__3uhm2ip8fgvd6t71aamlili27djecra7poo593emtkqovgn31ej91mbv5rijrcrv/f/8f813cb3d88c42c28ab26a624527777a as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/8133bf0180eb47986f00610092ee73e6/f/6ac106289c2c452bbc9655ffb80da128_SeqId_4_ 2024-11-24T04:23:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ffa06b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:44,851 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:44,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:44,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:44,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528f3fb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:44,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:44,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:44,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:44,853 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:44,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e02fb43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:44,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:44,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3c42d269 2024-11-24T04:23:44,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:44,855 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:23:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:44,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:44,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:44,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:23:44,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:44,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:44,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:44,870 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347. 2024-11-24T04:23:44,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:44,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:44,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:44,870 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:44,871 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__ns2_table2__3uhm2ip8fgvd6t71aamlili27djecra7poo593emtkqovgn31ej91mbv5rijrcrv/f/8f813cb3d88c42c28ab26a624527777a into 8133bf0180eb47986f00610092ee73e6/f as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/8133bf0180eb47986f00610092ee73e6/f/6ac106289c2c452bbc9655ffb80da128_SeqId_4_ - updating store file list. 2024-11-24T04:23:44,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/8133bf0180eb47986f00610092ee73e6/f/6ac106289c2c452bbc9655ffb80da128_SeqId_4_ into 8133bf0180eb47986f00610092ee73e6/f 2024-11-24T04:23:44,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__ns2_table2__3uhm2ip8fgvd6t71aamlili27djecra7poo593emtkqovgn31ej91mbv5rijrcrv/f/8f813cb3d88c42c28ab26a624527777a into 8133bf0180eb47986f00610092ee73e6/f (new location: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/8133bf0180eb47986f00610092ee73e6/f/6ac106289c2c452bbc9655ffb80da128_SeqId_4_) 2024-11-24T04:23:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/staging/jenkins__ns2_table2__3uhm2ip8fgvd6t71aamlili27djecra7poo593emtkqovgn31ej91mbv5rijrcrv/f/8f813cb3d88c42c28ab26a624527777a 2024-11-24T04:23:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e3e5a59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ClusterIdFetcher(90): Going to request e57b7116ff5e,46719,-1 for getting cluster id 2024-11-24T04:23:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:23:44,878 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b0ce8b-4af7-45ef-973c-2ebbd06519bb' 2024-11-24T04:23:44,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:23:44,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b0ce8b-4af7-45ef-973c-2ebbd06519bb" 2024-11-24T04:23:44,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a675123, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:44,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e57b7116ff5e,46719,-1] 2024-11-24T04:23:44,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:23:44,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:44,879 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:23:44,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@591274ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T04:23:44,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:23:44,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:44,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6b9eab11 2024-11-24T04:23:44,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:23:44,881 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T04:23:44,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:23:44,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e57b7116ff5e,36347,1732422095008, seqNum=-1] 2024-11-24T04:23:44,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:23:44,883 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T04:23:44,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T04:23:44,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T04:23:44,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:44,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347. 2024-11-24T04:23:44,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:44,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:44,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:44,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:44,898 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:23:44,899 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:44,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:44,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:44,899 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:44,899 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-11-24T04:23:44,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 , the old value is region=ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=e57b7116ff5e:36347 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T04:23:44,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T04:23:44,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 from cache 2024-11-24T04:23:44,900 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-24T04:23:44,913 INFO [Time-limited test {}] impl.RestoreTablesClient(175): No incremental changes since full backup for 'ns2:test-17324221039661', skipping incremental restore step. 2024-11-24T04:23:44,913 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-11-24T04:23:44,913 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1732422186353 hdfs://localhost:45347/backupUT/backup_1732422186353/ns2/test-17324221039661/ 2024-11-24T04:23:44,913 DEBUG [Time-limited test {}] impl.RestoreTablesClient(236): restoreStage finished 2024-11-24T04:23:44,915 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(217): After incremental restore: 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:23:44,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.CallRunner(138): callId: 742 service: ClientService methodName: Scan size: 160 connection: 172.17.0.3:45556 deadline: 1732422284914, exception=org.apache.hadoop.hbase.NotServingRegionException: table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. is not online on e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:44,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 , the old value is region=table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. is not online on e57b7116ff5e,36347,1732422095008 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:44,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e. is not online on e57b7116ff5e,36347,1732422095008 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:44,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1732422199699.a039ab1cc3e38eda24eec1b755eec11e., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 from cache 2024-11-24T04:23:45,011 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'ns2:table2', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=e57b7116ff5e:36347 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 34 more 2024-11-24T04:23:45,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., hostname=e57b7116ff5e,36347,1732422095008, seqNum=7] 2024-11-24T04:23:45,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.CallRunner(138): callId: 746 service: ClientService methodName: Scan size: 170 connection: 172.17.0.3:45556 deadline: 1732422285030, exception=org.apache.hadoop.hbase.NotServingRegionException: table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. is not online on e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:45,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 , the old value is region=table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. is not online on e57b7116ff5e,36347,1732422095008 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:45,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc. is not online on e57b7116ff5e,36347,1732422095008 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:45,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,row0,1732422199699.949a2f97f8cb299ab38ebab18f920edc., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 from cache 2024-11-24T04:23:45,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b., hostname=e57b7116ff5e,36347,1732422095008, seqNum=7] 2024-11-24T04:23:45,150 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(219): f1 has 198 rows 2024-11-24T04:23:45,156 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(223): f2 has 7 rows 2024-11-24T04:23:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:45,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-24T04:23:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:45,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-24T04:23:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:45,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-24T04:23:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:45,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-24T04:23:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:45,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-24T04:23:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:23:45,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-24T04:23:45,181 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(227): mob has 111 rows 2024-11-24T04:23:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36347 {}] ipc.CallRunner(138): callId: 754 service: ClientService methodName: Scan size: 159 connection: 172.17.0.3:45556 deadline: 1732422285181, exception=org.apache.hadoop.hbase.NotServingRegionException: ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. is not online on e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:45,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 , the old value is region=ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. is not online on e57b7116ff5e,36347,1732422095008 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:45,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779. is not online on e57b7116ff5e,36347,1732422095008 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T04:23:45,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2 from cache 2024-11-24T04:23:45,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6., hostname=e57b7116ff5e,36347,1732422095008, seqNum=2] 2024-11-24T04:23:45,296 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:23:45,296 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:235) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:45,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:45,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:45,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:45,314 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=633 (was 611) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: (e57b7116ff5e,46719,1732422093387)-proc-coordinator-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/cluster_bf592bf1-aea9-8622-9295-bd02d49cdf8d/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1911929303-172.17.0.3-1732422089355:blk_1073741954_1130, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: member: 'e57b7116ff5e,36347,1732422095008' subprocedure-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/cluster_bf592bf1-aea9-8622-9295-bd02d49cdf8d/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050297740_20 at /127.0.0.1:40056 [Receiving block BP-1911929303-172.17.0.3-1732422089355:blk_1073741954_1130] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1911929303-172.17.0.3-1732422089355:blk_1073741955_1131, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/e57b7116ff5e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'JobTracker' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(e57b7116ff5e,36347,1732422095008)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1544651422_20 at /127.0.0.1:52478 [Waiting for operation #54] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050297740_20 at /127.0.0.1:40058 [Receiving block BP-1911929303-172.17.0.3-1732422089355:blk_1073741955_1131] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/e57b7116ff5e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050297740_20 at /127.0.0.1:40046 [Receiving block BP-1911929303-172.17.0.3-1732422089355:blk_1073741953_1129] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1911929303-172.17.0.3-1732422089355:blk_1073741953_1129, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=641 (was 633) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=85 (was 207), ProcessCount=15 (was 15), AvailableMemoryMB=7606 (was 7990) 2024-11-24T04:23:45,314 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=633 is superior to 500 2024-11-24T04:23:45,321 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T04:23:45,321 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:23:45,321 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:45,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:45,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:45,321 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:45,321 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T04:23:45,322 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=27897051, stopped=false 2024-11-24T04:23:45,322 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e57b7116ff5e,46719,1732422093387 2024-11-24T04:23:45,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-24T04:23:45,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-24T04:23:45,370 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T04:23:45,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:23:45,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:23:45,370 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:23:45,371 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:45,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:45,371 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-24T04:23:45,371 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e57b7116ff5e,36347,1732422095008' ***** 2024-11-24T04:23:45,371 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T04:23:45,371 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-24T04:23:45,371 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-24T04:23:45,375 INFO [RS:0;e57b7116ff5e:36347 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@9984275{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-24T04:23:45,377 INFO [RS:0;e57b7116ff5e:36347 {}] server.AbstractConnector(383): Stopped ServerConnector@445a8045{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-24T04:23:45,377 INFO [RS:0;e57b7116ff5e:36347 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:23:45,377 INFO [RS:0;e57b7116ff5e:36347 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@2df3a35c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-24T04:23:45,377 INFO [RS:0;e57b7116ff5e:36347 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3e9615b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,STOPPED} 2024-11-24T04:23:45,379 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T04:23:45,379 INFO [RS:0;e57b7116ff5e:36347 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T04:23:45,379 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T04:23:45,379 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2024-11-24T04:23:45,379 INFO [RS:0;e57b7116ff5e:36347 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T04:23:45,379 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(3091): Received CLOSE for 5a030641752b7def7c38c5f6295501b2 2024-11-24T04:23:45,379 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(3091): Received CLOSE for 2c375e169f1afc2d9cce55928a15d594 2024-11-24T04:23:45,379 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(3091): Received CLOSE for a75e6c6417a138961442a072386b6bcc 2024-11-24T04:23:45,379 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(3091): Received CLOSE for a15495df03a7b8d0d73077c21d437b8b 2024-11-24T04:23:45,380 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(3091): Received CLOSE for 69e3c311ed1f46d175b4045bb296446a 2024-11-24T04:23:45,380 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(3091): Received CLOSE for 8133bf0180eb47986f00610092ee73e6 2024-11-24T04:23:45,380 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(3091): Received CLOSE for a35f3acd735a5c01c6819f7a642d051d 2024-11-24T04:23:45,380 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(3091): Received CLOSE for 4317ba5909dca438c339af86075fb75a 2024-11-24T04:23:45,380 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(3091): Received CLOSE for d3f34500755154053262a8d9790ac0a1 2024-11-24T04:23:45,380 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(959): stopping server e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:45,380 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T04:23:45,380 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5a030641752b7def7c38c5f6295501b2, disabling compactions & flushes 2024-11-24T04:23:45,380 INFO [RS:0;e57b7116ff5e:36347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e57b7116ff5e:36347. 2024-11-24T04:23:45,380 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:23:45,380 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:23:45,380 DEBUG [RS:0;e57b7116ff5e:36347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:45,380 DEBUG [RS:0;e57b7116ff5e:36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:45,380 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. after waiting 0 ms 2024-11-24T04:23:45,380 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:23:45,380 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5a030641752b7def7c38c5f6295501b2 2/2 column families, dataSize=2.12 KB heapSize=3.43 KB 2024-11-24T04:23:45,380 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T04:23:45,380 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T04:23:45,380 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T04:23:45,380 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T04:23:45,381 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-11-24T04:23:45,381 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T04:23:45,381 INFO [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T04:23:45,381 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T04:23:45,381 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T04:23:45,381 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T04:23:45,381 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2c375e169f1afc2d9cce55928a15d594, 4317ba5909dca438c339af86075fb75a, 5a030641752b7def7c38c5f6295501b2, 69e3c311ed1f46d175b4045bb296446a, 8133bf0180eb47986f00610092ee73e6, a15495df03a7b8d0d73077c21d437b8b, a35f3acd735a5c01c6819f7a642d051d, a75e6c6417a138961442a072386b6bcc, d3f34500755154053262a8d9790ac0a1 2024-11-24T04:23:45,381 INFO [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=23.97 KB heapSize=42.40 KB 2024-11-24T04:23:45,384 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/meta/270dd66c86d34a268f09b215eedd9599 is 158, key is trslm:hdfs://localhost:45347/backupUT\x00ns2:test-17324221039661/meta:log-roll-map/1732422199200/Put/seqid=0 2024-11-24T04:23:45,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741995_1171 (size=6208) 2024-11-24T04:23:45,403 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/.tmp/info/68eabeb0f1f8483c97a935f3585c41a5 is 159, key is ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1./info:regioninfo/1732422108052/Put/seqid=0 2024-11-24T04:23:45,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741996_1172 (size=19093) 2024-11-24T04:23:45,414 INFO [regionserver/e57b7116ff5e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:23:45,581 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2c375e169f1afc2d9cce55928a15d594, 4317ba5909dca438c339af86075fb75a, 5a030641752b7def7c38c5f6295501b2, 69e3c311ed1f46d175b4045bb296446a, 8133bf0180eb47986f00610092ee73e6, a15495df03a7b8d0d73077c21d437b8b, a35f3acd735a5c01c6819f7a642d051d, a75e6c6417a138961442a072386b6bcc, d3f34500755154053262a8d9790ac0a1 2024-11-24T04:23:45,781 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2c375e169f1afc2d9cce55928a15d594, 4317ba5909dca438c339af86075fb75a, 5a030641752b7def7c38c5f6295501b2, 69e3c311ed1f46d175b4045bb296446a, 8133bf0180eb47986f00610092ee73e6, a15495df03a7b8d0d73077c21d437b8b, a35f3acd735a5c01c6819f7a642d051d, a75e6c6417a138961442a072386b6bcc, d3f34500755154053262a8d9790ac0a1 2024-11-24T04:23:45,790 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=944 B at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/meta/270dd66c86d34a268f09b215eedd9599 2024-11-24T04:23:45,806 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/session/f3d0a805d88f4014a1532ae9357d5e53 is 588, key is session:backup_1732422186353/session:context/1732422199645/Put/seqid=0 2024-11-24T04:23:45,808 INFO [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.86 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/.tmp/info/68eabeb0f1f8483c97a935f3585c41a5 2024-11-24T04:23:45,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741997_1173 (size=5689) 2024-11-24T04:23:45,827 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/.tmp/ns/47c7f3c01fc5418285ac179f1481f2a7 is 85, key is ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779./ns:/1732422220243/DeleteFamily/seqid=0 2024-11-24T04:23:45,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741998_1174 (size=5760) 2024-11-24T04:23:45,982 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2c375e169f1afc2d9cce55928a15d594, 4317ba5909dca438c339af86075fb75a, 5a030641752b7def7c38c5f6295501b2, 69e3c311ed1f46d175b4045bb296446a, 8133bf0180eb47986f00610092ee73e6, a15495df03a7b8d0d73077c21d437b8b, a35f3acd735a5c01c6819f7a642d051d, a75e6c6417a138961442a072386b6bcc, d3f34500755154053262a8d9790ac0a1 2024-11-24T04:23:46,061 INFO [regionserver/e57b7116ff5e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T04:23:46,061 INFO [regionserver/e57b7116ff5e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T04:23:46,182 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2c375e169f1afc2d9cce55928a15d594, 4317ba5909dca438c339af86075fb75a, 5a030641752b7def7c38c5f6295501b2, 69e3c311ed1f46d175b4045bb296446a, 8133bf0180eb47986f00610092ee73e6, a15495df03a7b8d0d73077c21d437b8b, a35f3acd735a5c01c6819f7a642d051d, a75e6c6417a138961442a072386b6bcc, d3f34500755154053262a8d9790ac0a1 2024-11-24T04:23:46,213 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.20 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/session/f3d0a805d88f4014a1532ae9357d5e53 2024-11-24T04:23:46,225 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/meta/270dd66c86d34a268f09b215eedd9599 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/270dd66c86d34a268f09b215eedd9599 2024-11-24T04:23:46,231 INFO [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=474 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/.tmp/ns/47c7f3c01fc5418285ac179f1481f2a7 2024-11-24T04:23:46,233 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/meta/270dd66c86d34a268f09b215eedd9599, entries=7, sequenceid=52, filesize=6.1 K 2024-11-24T04:23:46,234 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/.tmp/session/f3d0a805d88f4014a1532ae9357d5e53 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/f3d0a805d88f4014a1532ae9357d5e53 2024-11-24T04:23:46,240 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/session/f3d0a805d88f4014a1532ae9357d5e53, entries=2, sequenceid=52, filesize=5.6 K 2024-11-24T04:23:46,241 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~2.12 KB/2170, heapSize ~3.40 KB/3480, currentSize=0 B/0 for 5a030641752b7def7c38c5f6295501b2 in 861ms, sequenceid=52, compaction requested=true 2024-11-24T04:23:46,246 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system/5a030641752b7def7c38c5f6295501b2/recovered.edits/55.seqid, newMaxSeqId=55, maxSeqId=1 2024-11-24T04:23:46,246 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:23:46,246 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:23:46,246 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5a030641752b7def7c38c5f6295501b2: Waiting for close lock at 1732422225380Running coprocessor pre-close hooks at 1732422225380Disabling compacts and flushes for region at 1732422225380Disabling writes for close at 1732422225380Obtaining lock to block concurrent updates at 1732422225380Preparing flush snapshotting stores in 5a030641752b7def7c38c5f6295501b2 at 1732422225380Finished memstore snapshotting backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2., syncing WAL and waiting on mvcc, flushsize=dataSize=2170, getHeapSize=3480, getOffHeapSize=0, getCellsCount=11 at 1732422225380Flushing stores of backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. at 1732422225381 (+1 ms)Flushing 5a030641752b7def7c38c5f6295501b2/meta: creating writer at 1732422225381Flushing 5a030641752b7def7c38c5f6295501b2/meta: appending metadata at 1732422225383 (+2 ms)Flushing 5a030641752b7def7c38c5f6295501b2/meta: closing flushed file at 1732422225383Flushing 5a030641752b7def7c38c5f6295501b2/session: creating writer at 1732422225803 (+420 ms)Flushing 5a030641752b7def7c38c5f6295501b2/session: appending metadata at 1732422225806 (+3 ms)Flushing 5a030641752b7def7c38c5f6295501b2/session: closing flushed file at 1732422225806Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53a533ba: reopening flushed file at 1732422226224 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5998ce04: reopening flushed file at 1732422226233 (+9 ms)Finished flush of dataSize ~2.12 KB/2170, heapSize ~3.40 KB/3480, currentSize=0 B/0 for 5a030641752b7def7c38c5f6295501b2 in 861ms, sequenceid=52, compaction requested=true at 1732422226241 (+8 ms)Writing region close event to WAL at 1732422226242 (+1 ms)Running coprocessor post-close hooks at 1732422226246 (+4 ms)Closed at 1732422226246 2024-11-24T04:23:46,247 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1732422115384.5a030641752b7def7c38c5f6295501b2. 2024-11-24T04:23:46,247 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2c375e169f1afc2d9cce55928a15d594, disabling compactions & flushes 2024-11-24T04:23:46,247 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:23:46,247 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:23:46,247 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. after waiting 0 ms 2024-11-24T04:23:46,247 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:23:46,247 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2c375e169f1afc2d9cce55928a15d594 3/3 column families, dataSize=378 B heapSize=1.63 KB 2024-11-24T04:23:46,254 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/.tmp/rep_barrier/25e37bdd047446c18363536843d409d2 is 94, key is ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779./rep_barrier:/1732422220243/DeleteFamily/seqid=0 2024-11-24T04:23:46,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741999_1175 (size=5640) 2024-11-24T04:23:46,262 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/f2/5f3dcfd906ff4649a8fe5ab696330d26 is 58, key is row-test-1732422103966-2-0/f2:q1/1732422181343/Put/seqid=0 2024-11-24T04:23:46,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073742000_1176 (size=5464) 2024-11-24T04:23:46,382 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-24T04:23:46,383 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1325): Online Regions={2c375e169f1afc2d9cce55928a15d594=test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594., a75e6c6417a138961442a072386b6bcc=ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc., a15495df03a7b8d0d73077c21d437b8b=table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b., 69e3c311ed1f46d175b4045bb296446a=table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a., 1588230740=hbase:meta,,1.1588230740, 8133bf0180eb47986f00610092ee73e6=ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6., a35f3acd735a5c01c6819f7a642d051d=ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d., 4317ba5909dca438c339af86075fb75a=backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a., d3f34500755154053262a8d9790ac0a1=ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1.} 2024-11-24T04:23:46,383 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2c375e169f1afc2d9cce55928a15d594, 4317ba5909dca438c339af86075fb75a, 69e3c311ed1f46d175b4045bb296446a, 8133bf0180eb47986f00610092ee73e6, a15495df03a7b8d0d73077c21d437b8b, a35f3acd735a5c01c6819f7a642d051d, a75e6c6417a138961442a072386b6bcc, d3f34500755154053262a8d9790ac0a1 2024-11-24T04:23:46,583 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2c375e169f1afc2d9cce55928a15d594, 4317ba5909dca438c339af86075fb75a, 69e3c311ed1f46d175b4045bb296446a, 8133bf0180eb47986f00610092ee73e6, a15495df03a7b8d0d73077c21d437b8b, a35f3acd735a5c01c6819f7a642d051d, a75e6c6417a138961442a072386b6bcc, d3f34500755154053262a8d9790ac0a1 2024-11-24T04:23:46,660 INFO [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=266 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/.tmp/rep_barrier/25e37bdd047446c18363536843d409d2 2024-11-24T04:23:46,667 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=378 B at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/f2/5f3dcfd906ff4649a8fe5ab696330d26 2024-11-24T04:23:46,676 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/.tmp/f2/5f3dcfd906ff4649a8fe5ab696330d26 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f2/5f3dcfd906ff4649a8fe5ab696330d26 2024-11-24T04:23:46,682 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/f2/5f3dcfd906ff4649a8fe5ab696330d26, entries=7, sequenceid=348, filesize=5.3 K 2024-11-24T04:23:46,683 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 2c375e169f1afc2d9cce55928a15d594 in 436ms, sequenceid=348, compaction requested=false 2024-11-24T04:23:46,688 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/test-1732422103966/2c375e169f1afc2d9cce55928a15d594/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=337 2024-11-24T04:23:46,688 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:23:46,688 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:23:46,688 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2c375e169f1afc2d9cce55928a15d594: Waiting for close lock at 1732422226247Running coprocessor pre-close hooks at 1732422226247Disabling compacts and flushes for region at 1732422226247Disabling writes for close at 1732422226247Obtaining lock to block concurrent updates at 1732422226247Preparing flush snapshotting stores in 2c375e169f1afc2d9cce55928a15d594 at 1732422226247Finished memstore snapshotting test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594., syncing WAL and waiting on mvcc, flushsize=dataSize=378, getHeapSize=1616, getOffHeapSize=0, getCellsCount=7 at 1732422226247Flushing stores of test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. at 1732422226248 (+1 ms)Flushing 2c375e169f1afc2d9cce55928a15d594/f2: creating writer at 1732422226248Flushing 2c375e169f1afc2d9cce55928a15d594/f2: appending metadata at 1732422226261 (+13 ms)Flushing 2c375e169f1afc2d9cce55928a15d594/f2: closing flushed file at 1732422226261Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d961bb9: reopening flushed file at 1732422226675 (+414 ms)Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 2c375e169f1afc2d9cce55928a15d594 in 436ms, sequenceid=348, compaction requested=false at 1732422226683 (+8 ms)Writing region close event to WAL at 1732422226684 (+1 ms)Running coprocessor post-close hooks at 1732422226688 (+4 ms)Closed at 1732422226688 2024-11-24T04:23:46,688 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1732422103966,,1732422104456.2c375e169f1afc2d9cce55928a15d594. 2024-11-24T04:23:46,688 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a75e6c6417a138961442a072386b6bcc, disabling compactions & flushes 2024-11-24T04:23:46,688 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. 2024-11-24T04:23:46,689 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. 2024-11-24T04:23:46,689 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. after waiting 0 ms 2024-11-24T04:23:46,689 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. 2024-11-24T04:23:46,691 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/.tmp/table/37034393416c403e88a47615c90881ba is 88, key is ns2:table2,,1732422205514.589b5c4b38f159789612c7e3246e2779./table:/1732422220243/DeleteFamily/seqid=0 2024-11-24T04:23:46,694 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns4/test-17324221039663/a75e6c6417a138961442a072386b6bcc/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T04:23:46,695 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:23:46,695 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. 2024-11-24T04:23:46,695 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a75e6c6417a138961442a072386b6bcc: Waiting for close lock at 1732422226688Running coprocessor pre-close hooks at 1732422226688Disabling compacts and flushes for region at 1732422226688Disabling writes for close at 1732422226689 (+1 ms)Writing region close event to WAL at 1732422226689Running coprocessor post-close hooks at 1732422226695 (+6 ms)Closed at 1732422226695 2024-11-24T04:23:46,695 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17324221039663,,1732422111322.a75e6c6417a138961442a072386b6bcc. 2024-11-24T04:23:46,695 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a15495df03a7b8d0d73077c21d437b8b, disabling compactions & flushes 2024-11-24T04:23:46,695 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:46,695 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:46,695 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. after waiting 0 ms 2024-11-24T04:23:46,695 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:46,701 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/a15495df03a7b8d0d73077c21d437b8b/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=6 2024-11-24T04:23:46,701 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:23:46,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073742001_1177 (size=6364) 2024-11-24T04:23:46,701 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:46,702 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a15495df03a7b8d0d73077c21d437b8b: Waiting for close lock at 1732422226695Running coprocessor pre-close hooks at 1732422226695Disabling compacts and flushes for region at 1732422226695Disabling writes for close at 1732422226695Writing region close event to WAL at 1732422226697 (+2 ms)Running coprocessor post-close hooks at 1732422226701 (+4 ms)Closed at 1732422226701 2024-11-24T04:23:46,702 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,row0,1732422210632.a15495df03a7b8d0d73077c21d437b8b. 2024-11-24T04:23:46,702 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 69e3c311ed1f46d175b4045bb296446a, disabling compactions & flushes 2024-11-24T04:23:46,702 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:46,702 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:46,702 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. after waiting 0 ms 2024-11-24T04:23:46,702 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:46,709 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/default/table1/69e3c311ed1f46d175b4045bb296446a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=6 2024-11-24T04:23:46,709 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:23:46,709 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:46,709 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 69e3c311ed1f46d175b4045bb296446a: Waiting for close lock at 1732422226702Running coprocessor pre-close hooks at 1732422226702Disabling compacts and flushes for region at 1732422226702Disabling writes for close at 1732422226702Writing region close event to WAL at 1732422226706 (+4 ms)Running coprocessor post-close hooks at 1732422226709 (+3 ms)Closed at 1732422226709 2024-11-24T04:23:46,709 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1732422210632.69e3c311ed1f46d175b4045bb296446a. 2024-11-24T04:23:46,710 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8133bf0180eb47986f00610092ee73e6, disabling compactions & flushes 2024-11-24T04:23:46,710 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6. 2024-11-24T04:23:46,710 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6. 2024-11-24T04:23:46,710 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6. after waiting 0 ms 2024-11-24T04:23:46,710 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6. 2024-11-24T04:23:46,713 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/table2/8133bf0180eb47986f00610092ee73e6/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-24T04:23:46,714 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:23:46,714 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6. 2024-11-24T04:23:46,714 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8133bf0180eb47986f00610092ee73e6: Waiting for close lock at 1732422226710Running coprocessor pre-close hooks at 1732422226710Disabling compacts and flushes for region at 1732422226710Disabling writes for close at 1732422226710Writing region close event to WAL at 1732422226710Running coprocessor post-close hooks at 1732422226714 (+4 ms)Closed at 1732422226714 2024-11-24T04:23:46,714 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:table2,,1732422220242.8133bf0180eb47986f00610092ee73e6. 2024-11-24T04:23:46,714 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a35f3acd735a5c01c6819f7a642d051d, disabling compactions & flushes 2024-11-24T04:23:46,714 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. 2024-11-24T04:23:46,714 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. 2024-11-24T04:23:46,714 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. after waiting 0 ms 2024-11-24T04:23:46,714 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. 2024-11-24T04:23:46,718 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns3/test-17324221039662/a35f3acd735a5c01c6819f7a642d051d/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T04:23:46,718 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:23:46,718 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. 2024-11-24T04:23:46,718 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a35f3acd735a5c01c6819f7a642d051d: Waiting for close lock at 1732422226714Running coprocessor pre-close hooks at 1732422226714Disabling compacts and flushes for region at 1732422226714Disabling writes for close at 1732422226714Writing region close event to WAL at 1732422226715 (+1 ms)Running coprocessor post-close hooks at 1732422226718 (+3 ms)Closed at 1732422226718 2024-11-24T04:23:46,718 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17324221039662,,1732422109153.a35f3acd735a5c01c6819f7a642d051d. 2024-11-24T04:23:46,718 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4317ba5909dca438c339af86075fb75a, disabling compactions & flushes 2024-11-24T04:23:46,718 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. 2024-11-24T04:23:46,718 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. 2024-11-24T04:23:46,718 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. after waiting 0 ms 2024-11-24T04:23:46,718 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. 2024-11-24T04:23:46,721 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/backup/system_bulk/4317ba5909dca438c339af86075fb75a/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T04:23:46,722 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:23:46,722 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. 2024-11-24T04:23:46,722 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4317ba5909dca438c339af86075fb75a: Waiting for close lock at 1732422226718Running coprocessor pre-close hooks at 1732422226718Disabling compacts and flushes for region at 1732422226718Disabling writes for close at 1732422226718Writing region close event to WAL at 1732422226719 (+1 ms)Running coprocessor post-close hooks at 1732422226722 (+3 ms)Closed at 1732422226722 2024-11-24T04:23:46,722 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1732422117558.4317ba5909dca438c339af86075fb75a. 2024-11-24T04:23:46,722 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d3f34500755154053262a8d9790ac0a1, disabling compactions & flushes 2024-11-24T04:23:46,722 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:23:46,722 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:23:46,722 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. after waiting 0 ms 2024-11-24T04:23:46,722 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:23:46,725 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/ns2/test-17324221039661/d3f34500755154053262a8d9790ac0a1/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2024-11-24T04:23:46,725 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:23:46,725 INFO [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:23:46,725 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d3f34500755154053262a8d9790ac0a1: Waiting for close lock at 1732422226722Running coprocessor pre-close hooks at 1732422226722Disabling compacts and flushes for region at 1732422226722Disabling writes for close at 1732422226722Writing region close event to WAL at 1732422226723 (+1 ms)Running coprocessor post-close hooks at 1732422226725 (+2 ms)Closed at 1732422226725 2024-11-24T04:23:46,726 DEBUG [RS_CLOSE_REGION-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17324221039661,,1732422106850.d3f34500755154053262a8d9790ac0a1. 2024-11-24T04:23:46,783 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T04:23:46,983 DEBUG [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T04:23:47,103 INFO [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.39 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/.tmp/table/37034393416c403e88a47615c90881ba 2024-11-24T04:23:47,118 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/.tmp/info/68eabeb0f1f8483c97a935f3585c41a5 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/info/68eabeb0f1f8483c97a935f3585c41a5 2024-11-24T04:23:47,125 INFO [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/info/68eabeb0f1f8483c97a935f3585c41a5, entries=114, sequenceid=97, filesize=18.6 K 2024-11-24T04:23:47,126 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/.tmp/ns/47c7f3c01fc5418285ac179f1481f2a7 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/ns/47c7f3c01fc5418285ac179f1481f2a7 2024-11-24T04:23:47,132 INFO [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/ns/47c7f3c01fc5418285ac179f1481f2a7, entries=10, sequenceid=97, filesize=5.6 K 2024-11-24T04:23:47,133 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/.tmp/rep_barrier/25e37bdd047446c18363536843d409d2 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/rep_barrier/25e37bdd047446c18363536843d409d2 2024-11-24T04:23:47,139 INFO [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/rep_barrier/25e37bdd047446c18363536843d409d2, entries=3, sequenceid=97, filesize=5.5 K 2024-11-24T04:23:47,140 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/.tmp/table/37034393416c403e88a47615c90881ba as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/table/37034393416c403e88a47615c90881ba 2024-11-24T04:23:47,146 INFO [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/table/37034393416c403e88a47615c90881ba, entries=21, sequenceid=97, filesize=6.2 K 2024-11-24T04:23:47,147 INFO [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~23.97 KB/24544, heapSize ~42.34 KB/43352, currentSize=0 B/0 for 1588230740 in 1766ms, sequenceid=97, compaction requested=false 2024-11-24T04:23:47,152 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/data/hbase/meta/1588230740/recovered.edits/100.seqid, newMaxSeqId=100, maxSeqId=1 2024-11-24T04:23:47,153 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T04:23:47,153 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T04:23:47,153 INFO [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T04:23:47,153 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732422225381Running coprocessor pre-close hooks at 1732422225381Disabling compacts and flushes for region at 1732422225381Disabling writes for close at 1732422225381Obtaining lock to block concurrent updates at 1732422225381Preparing flush snapshotting stores in 1588230740 at 1732422225381Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=24544, getHeapSize=43352, getOffHeapSize=0, getCellsCount=235 at 1732422225381Flushing stores of hbase:meta,,1.1588230740 at 1732422225382 (+1 ms)Flushing 1588230740/info: creating writer at 1732422225382Flushing 1588230740/info: appending metadata at 1732422225403 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732422225403Flushing 1588230740/ns: creating writer at 1732422225813 (+410 ms)Flushing 1588230740/ns: appending metadata at 1732422225827 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732422225827Flushing 1588230740/rep_barrier: creating writer at 1732422226238 (+411 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732422226254 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732422226254Flushing 1588230740/table: creating writer at 1732422226671 (+417 ms)Flushing 1588230740/table: appending metadata at 1732422226690 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732422226690Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37fea2c5: reopening flushed file at 1732422227117 (+427 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d9dadb4: reopening flushed file at 1732422227125 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dd5da7d: reopening flushed file at 1732422227132 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c0e12: reopening flushed file at 1732422227139 (+7 ms)Finished flush of dataSize ~23.97 KB/24544, heapSize ~42.34 KB/43352, currentSize=0 B/0 for 1588230740 in 1766ms, sequenceid=97, compaction requested=false at 1732422227147 (+8 ms)Writing region close event to WAL at 1732422227149 (+2 ms)Running coprocessor post-close hooks at 1732422227153 (+4 ms)Closed at 1732422227153 2024-11-24T04:23:47,153 DEBUG [RS_CLOSE_META-regionserver/e57b7116ff5e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T04:23:47,184 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(976): stopping server e57b7116ff5e,36347,1732422095008; all regions closed. 2024-11-24T04:23:47,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741954_1130 (size=16160) 2024-11-24T04:23:47,193 DEBUG [RS:0;e57b7116ff5e:36347 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs 2024-11-24T04:23:47,193 INFO [RS:0;e57b7116ff5e:36347 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e57b7116ff5e%2C36347%2C1732422095008.meta:.meta(num 1732422190909) 2024-11-24T04:23:47,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741953_1129 (size=4448) 2024-11-24T04:23:47,196 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0.1732422190889 not finished, retry = 0 2024-11-24T04:23:47,302 DEBUG [RS:0;e57b7116ff5e:36347 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs 2024-11-24T04:23:47,302 INFO [RS:0;e57b7116ff5e:36347 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-0:(num 1732422190889) 2024-11-24T04:23:47,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741955_1131 (size=7600) 2024-11-24T04:23:47,305 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/WALs/e57b7116ff5e,36347,1732422095008/e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1.1732422190925 not finished, retry = 0 2024-11-24T04:23:47,413 DEBUG [RS:0;e57b7116ff5e:36347 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/oldWALs 2024-11-24T04:23:47,413 INFO [RS:0;e57b7116ff5e:36347 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e57b7116ff5e%2C36347%2C1732422095008.e57b7116ff5e%2C36347%2C1732422095008.regiongroup-1:(num 1732422190925) 2024-11-24T04:23:47,413 DEBUG [RS:0;e57b7116ff5e:36347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:47,413 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:23:47,413 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T04:23:47,414 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.ChoreService(370): Chore service for: regionserver/e57b7116ff5e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T04:23:47,414 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T04:23:47,414 INFO [regionserver/e57b7116ff5e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T04:23:47,416 INFO [RS:0;e57b7116ff5e:36347 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36347 2024-11-24T04:23:47,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-24T04:23:47,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/e57b7116ff5e,36347,1732422095008 2024-11-24T04:23:47,465 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T04:23:47,475 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e57b7116ff5e,36347,1732422095008] 2024-11-24T04:23:47,485 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /1/draining/e57b7116ff5e,36347,1732422095008 already deleted, retry=false 2024-11-24T04:23:47,486 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e57b7116ff5e,36347,1732422095008 expired; onlineServers=0 2024-11-24T04:23:47,486 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e57b7116ff5e,46719,1732422093387' ***** 2024-11-24T04:23:47,486 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T04:23:47,486 INFO [M:0;e57b7116ff5e:46719 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-24T04:23:47,495 INFO [M:0;e57b7116ff5e:46719 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@5c685ba8{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-11-24T04:23:47,495 INFO [M:0;e57b7116ff5e:46719 {}] server.AbstractConnector(383): Stopped ServerConnector@10f9ba48{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-24T04:23:47,496 INFO [M:0;e57b7116ff5e:46719 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:23:47,496 INFO [M:0;e57b7116ff5e:46719 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@65507143{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-24T04:23:47,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2024-11-24T04:23:47,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T04:23:47,496 INFO [M:0;e57b7116ff5e:46719 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@7c1597dd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,STOPPED} 2024-11-24T04:23:47,496 INFO [M:0;e57b7116ff5e:46719 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T04:23:47,497 INFO [M:0;e57b7116ff5e:46719 {}] master.HMaster(1770): Stopping master jetty server 2024-11-24T04:23:47,497 INFO [M:0;e57b7116ff5e:46719 {}] server.AbstractConnector(383): Stopped ServerConnector@234e862b{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-24T04:23:47,498 INFO [M:0;e57b7116ff5e:46719 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T04:23:47,498 DEBUG [M:0;e57b7116ff5e:46719 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T04:23:47,498 INFO [M:0;e57b7116ff5e:46719 {}] master.BackupLogCleaner(194): Stopping BackupLogCleaner 2024-11-24T04:23:47,498 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T04:23:47,498 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster-HFileCleaner.small.0-1732422097828 {}] cleaner.HFileCleaner(306): Exit Thread[master/e57b7116ff5e:0:becomeActiveMaster-HFileCleaner.small.0-1732422097828,5,FailOnTimeoutGroup] 2024-11-24T04:23:47,498 DEBUG [master/e57b7116ff5e:0:becomeActiveMaster-HFileCleaner.large.0-1732422097826 {}] cleaner.HFileCleaner(306): Exit Thread[master/e57b7116ff5e:0:becomeActiveMaster-HFileCleaner.large.0-1732422097826,5,FailOnTimeoutGroup] 2024-11-24T04:23:47,498 INFO [M:0;e57b7116ff5e:46719 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;e57b7116ff5e:46719. 2024-11-24T04:23:47,498 DEBUG [M:0;e57b7116ff5e:46719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupHFileCleaner.stop(BackupHFileCleaner.java:156) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.cleanup(CleanerChore.java:444) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.cleanup(HFileCleaner.java:239) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:333) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:322) at org.apache.hadoop.hbase.master.HMaster.stopChores(HMaster.java:1923) at org.apache.hadoop.hbase.HBaseServerBase.stopChoreService(HBaseServerBase.java:439) at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1777) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:23:47,499 DEBUG [M:0;e57b7116ff5e:46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:47,499 DEBUG [M:0;e57b7116ff5e:46719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:23:47,499 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:23:47,499 DEBUG [M:0;e57b7116ff5e:46719 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T04:23:47,499 INFO [M:0;e57b7116ff5e:46719 {}] hbase.ChoreService(370): Chore service for: master/e57b7116ff5e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T04:23:47,499 INFO [M:0;e57b7116ff5e:46719 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T04:23:47,499 DEBUG [M:0;e57b7116ff5e:46719 {}] master.HMaster(1795): Stopping service threads 2024-11-24T04:23:47,499 INFO [M:0;e57b7116ff5e:46719 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T04:23:47,499 INFO [M:0;e57b7116ff5e:46719 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T04:23:47,500 ERROR [M:0;e57b7116ff5e:46719 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-1,5,PEWorkerGroup] Thread[HFileArchiver-2,5,PEWorkerGroup] Thread[HFileArchiver-3,5,PEWorkerGroup] 2024-11-24T04:23:47,500 INFO [M:0;e57b7116ff5e:46719 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T04:23:47,500 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T04:23:47,501 DEBUG [M:0;e57b7116ff5e:46719 {}] zookeeper.ZKUtil(347): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2024-11-24T04:23:47,501 WARN [M:0;e57b7116ff5e:46719 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T04:23:47,502 INFO [M:0;e57b7116ff5e:46719 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/.lastflushedseqids 2024-11-24T04:23:47,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073742002_1178 (size=761) 2024-11-24T04:23:47,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:23:47,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36347-0x1016b1342860001, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:23:47,576 INFO [RS:0;e57b7116ff5e:36347 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T04:23:47,577 INFO [RS:0;e57b7116ff5e:36347 {}] regionserver.HRegionServer(1031): Exiting; stopping=e57b7116ff5e,36347,1732422095008; zookeeper connection closed. 2024-11-24T04:23:47,577 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@673b33d5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@673b33d5 2024-11-24T04:23:47,578 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-24T04:23:47,912 INFO [M:0;e57b7116ff5e:46719 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T04:23:47,912 INFO [M:0;e57b7116ff5e:46719 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T04:23:47,913 DEBUG [M:0;e57b7116ff5e:46719 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T04:23:47,927 INFO [M:0;e57b7116ff5e:46719 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:23:47,927 DEBUG [M:0;e57b7116ff5e:46719 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:23:47,927 DEBUG [M:0;e57b7116ff5e:46719 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T04:23:47,927 DEBUG [M:0;e57b7116ff5e:46719 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:23:47,928 INFO [M:0;e57b7116ff5e:46719 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=436.61 KB heapSize=512.90 KB 2024-11-24T04:23:47,946 DEBUG [M:0;e57b7116ff5e:46719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/83d184678c1449d396757722381fbae6 is 82, key is hbase:meta,,1/info:regioninfo/1732422098763/Put/seqid=0 2024-11-24T04:23:47,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073742003_1179 (size=5672) 2024-11-24T04:23:48,351 INFO [M:0;e57b7116ff5e:46719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=898 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/83d184678c1449d396757722381fbae6 2024-11-24T04:23:48,380 DEBUG [M:0;e57b7116ff5e:46719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ffb5157ef55a41db8e2d1d96fd6cd96c is 2200, key is \x00\x00\x00\x00\x00\x00\x00P/proc:d/1732422217361/Put/seqid=0 2024-11-24T04:23:48,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073742004_1180 (size=28866) 2024-11-24T04:23:48,786 INFO [M:0;e57b7116ff5e:46719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=436.05 KB at sequenceid=898 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ffb5157ef55a41db8e2d1d96fd6cd96c 2024-11-24T04:23:48,797 INFO [M:0;e57b7116ff5e:46719 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ffb5157ef55a41db8e2d1d96fd6cd96c 2024-11-24T04:23:48,816 DEBUG [M:0;e57b7116ff5e:46719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9de5e4eceb4a473aa21254af8fb05315 is 69, key is e57b7116ff5e,36347,1732422095008/rs:state/1732422097937/Put/seqid=0 2024-11-24T04:23:48,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073742005_1181 (size=5156) 2024-11-24T04:23:49,221 INFO [M:0;e57b7116ff5e:46719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=898 (bloomFilter=true), to=hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9de5e4eceb4a473aa21254af8fb05315 2024-11-24T04:23:49,234 DEBUG [M:0;e57b7116ff5e:46719 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/83d184678c1449d396757722381fbae6 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/83d184678c1449d396757722381fbae6 2024-11-24T04:23:49,240 INFO [M:0;e57b7116ff5e:46719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/83d184678c1449d396757722381fbae6, entries=8, sequenceid=898, filesize=5.5 K 2024-11-24T04:23:49,241 DEBUG [M:0;e57b7116ff5e:46719 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ffb5157ef55a41db8e2d1d96fd6cd96c as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ffb5157ef55a41db8e2d1d96fd6cd96c 2024-11-24T04:23:49,247 INFO [M:0;e57b7116ff5e:46719 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ffb5157ef55a41db8e2d1d96fd6cd96c 2024-11-24T04:23:49,247 INFO [M:0;e57b7116ff5e:46719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ffb5157ef55a41db8e2d1d96fd6cd96c, entries=93, sequenceid=898, filesize=28.2 K 2024-11-24T04:23:49,248 DEBUG [M:0;e57b7116ff5e:46719 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9de5e4eceb4a473aa21254af8fb05315 as hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9de5e4eceb4a473aa21254af8fb05315 2024-11-24T04:23:49,253 INFO [M:0;e57b7116ff5e:46719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45347/user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9de5e4eceb4a473aa21254af8fb05315, entries=1, sequenceid=898, filesize=5.0 K 2024-11-24T04:23:49,254 INFO [M:0;e57b7116ff5e:46719 {}] regionserver.HRegion(3140): Finished flush of dataSize ~436.61 KB/447087, heapSize ~512.60 KB/524904, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1327ms, sequenceid=898, compaction requested=false 2024-11-24T04:23:49,255 INFO [M:0;e57b7116ff5e:46719 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:23:49,256 DEBUG [M:0;e57b7116ff5e:46719 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732422227912Disabling compacts and flushes for region at 1732422227912Disabling writes for close at 1732422227927 (+15 ms)Obtaining lock to block concurrent updates at 1732422227928 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732422227928Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=447087, getHeapSize=525144, getOffHeapSize=0, getCellsCount=1021 at 1732422227928Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732422227928Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732422227928Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732422227945 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732422227945Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732422228362 (+417 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732422228380 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732422228380Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732422228797 (+417 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732422228815 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732422228815Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@582ebc0f: reopening flushed file at 1732422229233 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53aeb209: reopening flushed file at 1732422229240 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fc32fa5: reopening flushed file at 1732422229247 (+7 ms)Finished flush of dataSize ~436.61 KB/447087, heapSize ~512.60 KB/524904, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1327ms, sequenceid=898, compaction requested=false at 1732422229254 (+7 ms)Writing region close event to WAL at 1732422229255 (+1 ms)Closed at 1732422229255 2024-11-24T04:23:49,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44025 is added to blk_1073741830_1006 (size=506368) 2024-11-24T04:23:49,258 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2ad54eaa-8758-c8d7-3ee4-c34d92de0cbd/MasterData/WALs/e57b7116ff5e,46719,1732422093387/e57b7116ff5e%2C46719%2C1732422093387.e57b7116ff5e%2C46719%2C1732422093387.regiongroup-0.1732422096873 not finished, retry = 0 2024-11-24T04:23:49,360 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T04:23:49,360 INFO [M:0;e57b7116ff5e:46719 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T04:23:49,361 INFO [M:0;e57b7116ff5e:46719 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2024-11-24T04:23:49,361 INFO [M:0;e57b7116ff5e:46719 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46719 2024-11-24T04:23:49,361 INFO [M:0;e57b7116ff5e:46719 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T04:23:49,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:23:49,502 INFO [M:0;e57b7116ff5e:46719 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T04:23:49,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46719-0x1016b1342860000, quorum=127.0.0.1:64163, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:23:49,523 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@cb44ad4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:23:49,526 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d3ca681{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:23:49,526 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:23:49,527 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8cd91a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T04:23:49,527 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@526a6450{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,STOPPED} 2024-11-24T04:23:49,529 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T04:23:49,529 WARN [BP-1911929303-172.17.0.3-1732422089355 heartbeating to localhost/127.0.0.1:45347 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T04:23:49,529 WARN [BP-1911929303-172.17.0.3-1732422089355 heartbeating to localhost/127.0.0.1:45347 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1911929303-172.17.0.3-1732422089355 (Datanode Uuid b485fe1d-3b22-4447-85b3-8150d5328dec) service to localhost/127.0.0.1:45347 2024-11-24T04:23:49,529 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T04:23:49,531 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/cluster_bf592bf1-aea9-8622-9295-bd02d49cdf8d/data/data1/current/BP-1911929303-172.17.0.3-1732422089355 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:23:49,531 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/cluster_bf592bf1-aea9-8622-9295-bd02d49cdf8d/data/data2/current/BP-1911929303-172.17.0.3-1732422089355 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:23:49,531 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T04:23:49,537 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c784fd0{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T04:23:49,538 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2fdb04ed{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:23:49,538 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:23:49,538 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@282f8860{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T04:23:49,538 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3cb0c733{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,STOPPED} 2024-11-24T04:23:49,547 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T04:23:49,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T04:23:49,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2377): Stopping mini mapreduce cluster... 2024-11-24T04:23:49,615 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@43aedd1d{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-24T04:23:49,616 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c6d042f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:23:49,616 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:23:49,616 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@436a6a5d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-24T04:23:49,616 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d51f859{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,STOPPED} 2024-11-24T04:23:49,624 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@39275f7{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-24T04:23:49,625 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@159887a6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:23:49,625 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:23:49,625 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@722bd21f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-24T04:23:49,625 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bdc2c31{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,STOPPED} 2024-11-24T04:23:49,635 ERROR [Thread[Thread-270,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-24T04:23:49,636 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@52227462{cluster,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-24T04:23:49,636 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@382d96c0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:23:49,636 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:23:49,636 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11a358cb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-24T04:23:49,636 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@339da2c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,STOPPED} 2024-11-24T04:23:49,638 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2024-11-24T04:23:49,641 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(72): Returning, interrupted : java.lang.InterruptedException 2024-11-24T04:23:49,641 ERROR [ResourceManager Event Processor Monitor {}] resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessorMonitor(1193): Returning, interrupted : java.lang.InterruptedException: sleep interrupted 2024-11-24T04:23:49,642 ERROR [Thread[Thread-293,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-24T04:23:49,645 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5ca13ded{jobhistory,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-24T04:23:49,645 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7eba726a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:23:49,645 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:23:49,646 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2860dd4f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-24T04:23:49,646 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@37164ab0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/0cfcd03f-d95b-8713-8ad9-1054f6910cef/hadoop.log.dir/,STOPPED} 2024-11-24T04:23:49,647 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-24T04:23:49,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2380): Mini mapreduce cluster stopped