2024-11-25 14:22:18,546 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec
2024-11-25 14:22:18,562 main DEBUG Took 0.013488 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-25 14:22:18,562 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-25 14:22:18,562 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-25 14:22:18,563 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-25 14:22:18,565 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 14:22:18,584 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-25 14:22:18,601 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 14:22:18,602 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 14:22:18,603 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 14:22:18,603 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 14:22:18,604 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 14:22:18,604 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 14:22:18,605 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 14:22:18,605 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 14:22:18,606 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 14:22:18,606 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 14:22:18,607 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 14:22:18,607 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 14:22:18,608 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 14:22:18,608 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 14:22:18,608 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 14:22:18,609 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 14:22:18,609 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 14:22:18,610 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 14:22:18,610 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 14:22:18,610 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 14:22:18,611 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 14:22:18,611 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 14:22:18,612 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 14:22:18,612 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 14:22:18,612 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 14:22:18,613 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-25 14:22:18,616 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 14:22:18,618 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-25 14:22:18,620 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-25 14:22:18,620 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-25 14:22:18,622 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-25 14:22:18,622 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-25 14:22:18,632 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-25 14:22:18,635 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-25 14:22:18,637 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-25 14:22:18,637 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-25 14:22:18,638 main DEBUG createAppenders(={Console})
2024-11-25 14:22:18,638 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec initialized
2024-11-25 14:22:18,639 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec
2024-11-25 14:22:18,639 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec OK.
2024-11-25 14:22:18,640 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-25 14:22:18,640 main DEBUG OutputStream closed
2024-11-25 14:22:18,640 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-25 14:22:18,641 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-25 14:22:18,641 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4efc180e OK
2024-11-25 14:22:18,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-25 14:22:18,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-25 14:22:18,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-25 14:22:18,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-25 14:22:18,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-25 14:22:18,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-25 14:22:18,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-25 14:22:18,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-25 14:22:18,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-25 14:22:18,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-25 14:22:18,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-25 14:22:18,735 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-25 14:22:18,735 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-25 14:22:18,735 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-25 14:22:18,736 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-25 14:22:18,736 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-25 14:22:18,736 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-25 14:22:18,737 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-25 14:22:18,740 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-25 14:22:18,740 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@39de3d36) with optional ClassLoader: null
2024-11-25 14:22:18,741 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-25 14:22:18,742 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@39de3d36] started OK.
2024-11-25T14:22:19,034 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91
2024-11-25 14:22:19,037 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-25 14:22:19,037 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-25T14:22:19,072 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi timeout: 26 mins
2024-11-25T14:22:19,098 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-25T14:22:19,122 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/cluster_f33e2636-644a-9dd7-bdf8-442b0e715513, deleteOnExit=true
2024-11-25T14:22:19,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-25T14:22:19,126 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/test.cache.data in system properties and HBase conf
2024-11-25T14:22:19,127 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/hadoop.tmp.dir in system properties and HBase conf
2024-11-25T14:22:19,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/hadoop.log.dir in system properties and HBase conf
2024-11-25T14:22:19,129 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-25T14:22:19,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-25T14:22:19,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-25T14:22:19,250 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-25T14:22:19,392 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-25T14:22:19,404 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-25T14:22:19,405 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-25T14:22:19,406 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-25T14:22:19,407 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-25T14:22:19,408 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-25T14:22:19,410 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-25T14:22:19,412 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-25T14:22:19,413 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-25T14:22:19,413 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-25T14:22:19,414 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/nfs.dump.dir in system properties and HBase conf
2024-11-25T14:22:19,415 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/java.io.tmpdir in system properties and HBase conf
2024-11-25T14:22:19,415 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-25T14:22:19,416 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-25T14:22:19,417 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-25T14:22:20,572 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-25T14:22:20,650 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2869ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-25T14:22:20,721 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-25T14:22:20,800 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-25T14:22:20,826 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-25T14:22:20,826 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-25T14:22:20,827 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-25T14:22:20,843 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-25T14:22:20,848 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@500f17d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/hadoop.log.dir/,AVAILABLE}
2024-11-25T14:22:20,849 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7cc143ab{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-25T14:22:21,083 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@38149812{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/java.io.tmpdir/jetty-localhost-45277-hadoop-hdfs-3_4_1-tests_jar-_-any-11421225299851301394/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-25T14:22:21,092 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17efcf28{HTTP/1.1, (http/1.1)}{localhost:45277}
2024-11-25T14:22:21,093 INFO  [Time-limited test {}] server.Server(415): Started @3313ms
2024-11-25T14:22:21,621 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-25T14:22:21,633 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-25T14:22:21,636 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-25T14:22:21,636 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-25T14:22:21,636 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-25T14:22:21,637 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5049cee1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/hadoop.log.dir/,AVAILABLE}
2024-11-25T14:22:21,638 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4043ca1d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-25T14:22:21,764 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2e64f7c7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/java.io.tmpdir/jetty-localhost-40097-hadoop-hdfs-3_4_1-tests_jar-_-any-9905842035023253118/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-25T14:22:21,765 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@577c436c{HTTP/1.1, (http/1.1)}{localhost:40097}
2024-11-25T14:22:21,765 INFO  [Time-limited test {}] server.Server(415): Started @3985ms
2024-11-25T14:22:21,829 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-25T14:22:22,802 WARN  [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/cluster_f33e2636-644a-9dd7-bdf8-442b0e715513/data/data1/current/BP-610877534-172.17.0.3-1732544540075/current, will proceed with Du for space computation calculation, 
2024-11-25T14:22:22,802 WARN  [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/cluster_f33e2636-644a-9dd7-bdf8-442b0e715513/data/data2/current/BP-610877534-172.17.0.3-1732544540075/current, will proceed with Du for space computation calculation, 
2024-11-25T14:22:22,844 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-25T14:22:22,888 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x397ec478db5b3437 with lease ID 0x8ac980f3e57c45dc: Processing first storage report for DS-1aab8e27-0a6e-4df2-9f24-0e782c02ea10 from datanode DatanodeRegistration(127.0.0.1:37535, datanodeUuid=71803459-73b8-4a49-97a8-eed8a0826365, infoPort=45247, infoSecurePort=0, ipcPort=40405, storageInfo=lv=-57;cid=testClusterID;nsid=2011211161;c=1732544540075)
2024-11-25T14:22:22,889 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x397ec478db5b3437 with lease ID 0x8ac980f3e57c45dc: from storage DS-1aab8e27-0a6e-4df2-9f24-0e782c02ea10 node DatanodeRegistration(127.0.0.1:37535, datanodeUuid=71803459-73b8-4a49-97a8-eed8a0826365, infoPort=45247, infoSecurePort=0, ipcPort=40405, storageInfo=lv=-57;cid=testClusterID;nsid=2011211161;c=1732544540075), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-11-25T14:22:22,890 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x397ec478db5b3437 with lease ID 0x8ac980f3e57c45dc: Processing first storage report for DS-c226a97a-87f4-4eec-8bd7-431213de2391 from datanode DatanodeRegistration(127.0.0.1:37535, datanodeUuid=71803459-73b8-4a49-97a8-eed8a0826365, infoPort=45247, infoSecurePort=0, ipcPort=40405, storageInfo=lv=-57;cid=testClusterID;nsid=2011211161;c=1732544540075)
2024-11-25T14:22:22,890 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x397ec478db5b3437 with lease ID 0x8ac980f3e57c45dc: from storage DS-c226a97a-87f4-4eec-8bd7-431213de2391 node DatanodeRegistration(127.0.0.1:37535, datanodeUuid=71803459-73b8-4a49-97a8-eed8a0826365, infoPort=45247, infoSecurePort=0, ipcPort=40405, storageInfo=lv=-57;cid=testClusterID;nsid=2011211161;c=1732544540075), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-25T14:22:22,898 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91
2024-11-25T14:22:22,966 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/cluster_f33e2636-644a-9dd7-bdf8-442b0e715513/zookeeper_0, clientPort=51135, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/cluster_f33e2636-644a-9dd7-bdf8-442b0e715513/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/cluster_f33e2636-644a-9dd7-bdf8-442b0e715513/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-25T14:22:22,974 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51135
2024-11-25T14:22:22,988 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-25T14:22:22,991 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-25T14:22:23,258 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741825_1001 (size=7)
2024-11-25T14:22:23,664 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581 with version=8
2024-11-25T14:22:23,664 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/hbase-staging
2024-11-25T14:22:23,754 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-25T14:22:23,988 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/cb69081f5976:0 server-side Connection retries=6
2024-11-25T14:22:24,000 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-25T14:22:24,001 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-25T14:22:24,010 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-25T14:22:24,011 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-25T14:22:24,011 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-25T14:22:24,192 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-25T14:22:24,257 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-25T14:22:24,266 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-25T14:22:24,269 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T14:22:24,298 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 48811 (auto-detected)
2024-11-25T14:22:24,299 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected)
2024-11-25T14:22:24,319 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44721
2024-11-25T14:22:24,342 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44721 connecting to ZooKeeper ensemble=127.0.0.1:51135
2024-11-25T14:22:24,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:447210x0, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-25T14:22:24,444 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44721-0x10173a96ace0000 connected
2024-11-25T14:22:24,546 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-25T14:22:24,549 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-25T14:22:24,558 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-25T14:22:24,562 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581, hbase.cluster.distributed=false
2024-11-25T14:22:24,586 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-25T14:22:24,595 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44721
2024-11-25T14:22:24,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44721
2024-11-25T14:22:24,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44721
2024-11-25T14:22:24,600 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44721
2024-11-25T14:22:24,600 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44721
2024-11-25T14:22:24,738 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/cb69081f5976:0 server-side Connection retries=6
2024-11-25T14:22:24,741 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-25T14:22:24,741 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-25T14:22:24,741 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-25T14:22:24,743 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-25T14:22:24,744 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-25T14:22:24,748 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-25T14:22:24,752 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T14:22:24,753 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36701
2024-11-25T14:22:24,760 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36701 connecting to ZooKeeper ensemble=127.0.0.1:51135
2024-11-25T14:22:24,762 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-25T14:22:24,765 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-25T14:22:24,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:367010x0, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-25T14:22:24,798 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:367010x0, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-25T14:22:24,802 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36701-0x10173a96ace0001 connected
2024-11-25T14:22:24,803 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-25T14:22:24,813 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-25T14:22:24,815 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-25T14:22:24,824 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-25T14:22:24,825 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36701
2024-11-25T14:22:24,825 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36701
2024-11-25T14:22:24,825 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36701
2024-11-25T14:22:24,826 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36701
2024-11-25T14:22:24,826 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36701
2024-11-25T14:22:24,849 DEBUG [M:0;cb69081f5976:44721 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;cb69081f5976:44721
2024-11-25T14:22:24,850 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/cb69081f5976,44721,1732544543804
2024-11-25T14:22:24,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-25T14:22:24,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-25T14:22:24,865 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/cb69081f5976,44721,1732544543804
2024-11-25T14:22:24,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:22:24,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-25T14:22:24,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:22:24,896 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-25T14:22:24,897 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/cb69081f5976,44721,1732544543804 from backup master directory
2024-11-25T14:22:24,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/cb69081f5976,44721,1732544543804
2024-11-25T14:22:24,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-25T14:22:24,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-25T14:22:24,911 WARN  [master/cb69081f5976:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-25T14:22:24,911 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=cb69081f5976,44721,1732544543804
2024-11-25T14:22:24,914 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-25T14:22:24,916 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-25T14:22:24,987 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/hbase.id] with ID: b9c5d22a-5299-4650-a1d7-40bf093ed607
2024-11-25T14:22:24,987 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/.tmp/hbase.id
2024-11-25T14:22:24,997 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741826_1002 (size=42)
2024-11-25T14:22:25,399 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/.tmp/hbase.id]:[hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/hbase.id]
2024-11-25T14:22:25,451 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-25T14:22:25,456 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-25T14:22:25,474 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms.
2024-11-25T14:22:25,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:22:25,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:22:25,526 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741827_1003 (size=196)
2024-11-25T14:22:25,942 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-25T14:22:25,945 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-25T14:22:25,960 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396
java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo)
	at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.<clinit>(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Class.forName0(Native Method) ~[?:?]
	at java.lang.Class.forName(Class.java:375) ~[?:?]
	at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:262) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:231) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:25,965 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-25T14:22:25,997 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741828_1004 (size=1189)
2024-11-25T14:22:26,415 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store
2024-11-25T14:22:26,436 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741829_1005 (size=34)
2024-11-25T14:22:26,842 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-25T14:22:26,845 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-25T14:22:26,846 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-25T14:22:26,846 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-25T14:22:26,846 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-25T14:22:26,848 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-25T14:22:26,848 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-25T14:22:26,848 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-25T14:22:26,850 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732544546846Disabling compacts and flushes for region at 1732544546846Disabling writes for close at 1732544546848 (+2 ms)Writing region close event to WAL at 1732544546848Closed at 1732544546848
2024-11-25T14:22:26,853 WARN  [master/cb69081f5976:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/.initializing
2024-11-25T14:22:26,853 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/WALs/cb69081f5976,44721,1732544543804
2024-11-25T14:22:26,862 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-11-25T14:22:26,878 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cb69081f5976%2C44721%2C1732544543804, suffix=, logDir=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/WALs/cb69081f5976,44721,1732544543804, archiveDir=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/oldWALs, maxLogs=10
2024-11-25T14:22:26,910 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/WALs/cb69081f5976,44721,1732544543804/cb69081f5976%2C44721%2C1732544543804.1732544546883, exclude list is [], retry=0
2024-11-25T14:22:26,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37535,DS-1aab8e27-0a6e-4df2-9f24-0e782c02ea10,DISK]
2024-11-25T14:22:26,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf.
2024-11-25T14:22:26,972 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/WALs/cb69081f5976,44721,1732544543804/cb69081f5976%2C44721%2C1732544543804.1732544546883
2024-11-25T14:22:26,974 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45247:45247)]
2024-11-25T14:22:26,974 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-25T14:22:26,975 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-25T14:22:26,979 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-25T14:22:26,980 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-25T14:22:27,025 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-25T14:22:27,050 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-25T14:22:27,053 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:27,056 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-25T14:22:27,057 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-25T14:22:27,060 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-25T14:22:27,060 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:27,062 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-25T14:22:27,062 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-25T14:22:27,065 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-25T14:22:27,066 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:27,067 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-25T14:22:27,067 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-25T14:22:27,071 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-25T14:22:27,071 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:27,072 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-25T14:22:27,073 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-25T14:22:27,077 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-25T14:22:27,079 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-25T14:22:27,086 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-25T14:22:27,087 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-25T14:22:27,091 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-25T14:22:27,095 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-25T14:22:27,100 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-25T14:22:27,102 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70170162, jitterRate=0.04561689496040344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-25T14:22:27,111 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732544546995Initializing all the Stores at 1732544546997 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732544546997Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732544546998 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732544546998Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732544546998Cleaning up temporary data from old regions at 1732544547087 (+89 ms)Region opened successfully at 1732544547111 (+24 ms)
2024-11-25T14:22:27,112 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-25T14:22:27,155 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14af86ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cb69081f5976/172.17.0.3:0
2024-11-25T14:22:27,197 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-25T14:22:27,210 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-25T14:22:27,211 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-25T14:22:27,214 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-25T14:22:27,215 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-25T14:22:27,221 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-25T14:22:27,221 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-25T14:22:27,250 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-25T14:22:27,261 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-25T14:22:27,268 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-25T14:22:27,271 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-25T14:22:27,274 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-25T14:22:27,284 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-25T14:22:27,287 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-25T14:22:27,291 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-25T14:22:27,301 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-25T14:22:27,303 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-25T14:22:27,318 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-25T14:22:27,340 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-25T14:22:27,351 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-25T14:22:27,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-25T14:22:27,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-25T14:22:27,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:22:27,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:22:27,362 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=cb69081f5976,44721,1732544543804, sessionid=0x10173a96ace0000, setting cluster-up flag (Was=false)
2024-11-25T14:22:27,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:22:27,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:22:27,409 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-25T14:22:27,411 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cb69081f5976,44721,1732544543804
2024-11-25T14:22:27,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:22:27,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:22:27,459 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-25T14:22:27,461 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cb69081f5976,44721,1732544543804
2024-11-25T14:22:27,468 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-25T14:22:27,533 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(746): ClusterId : b9c5d22a-5299-4650-a1d7-40bf093ed607
2024-11-25T14:22:27,536 DEBUG [RS:0;cb69081f5976:36701 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-25T14:22:27,545 DEBUG [RS:0;cb69081f5976:36701 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-25T14:22:27,545 DEBUG [RS:0;cb69081f5976:36701 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-25T14:22:27,553 DEBUG [RS:0;cb69081f5976:36701 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-25T14:22:27,553 DEBUG [RS:0;cb69081f5976:36701 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74d1006e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cb69081f5976/172.17.0.3:0
2024-11-25T14:22:27,557 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-25T14:22:27,569 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-25T14:22:27,569 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;cb69081f5976:36701
2024-11-25T14:22:27,573 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-25T14:22:27,573 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-25T14:22:27,573 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-25T14:22:27,576 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(2659): reportForDuty to master=cb69081f5976,44721,1732544543804 with port=36701, startcode=1732544544686
2024-11-25T14:22:27,576 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-25T14:22:27,583 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: cb69081f5976,44721,1732544543804
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-25T14:22:27,588 DEBUG [RS:0;cb69081f5976:36701 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-25T14:22:27,591 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/cb69081f5976:0, corePoolSize=5, maxPoolSize=5
2024-11-25T14:22:27,591 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/cb69081f5976:0, corePoolSize=5, maxPoolSize=5
2024-11-25T14:22:27,591 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/cb69081f5976:0, corePoolSize=5, maxPoolSize=5
2024-11-25T14:22:27,591 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/cb69081f5976:0, corePoolSize=5, maxPoolSize=5
2024-11-25T14:22:27,591 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/cb69081f5976:0, corePoolSize=10, maxPoolSize=10
2024-11-25T14:22:27,592 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/cb69081f5976:0, corePoolSize=1, maxPoolSize=1
2024-11-25T14:22:27,592 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/cb69081f5976:0, corePoolSize=2, maxPoolSize=2
2024-11-25T14:22:27,592 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/cb69081f5976:0, corePoolSize=1, maxPoolSize=1
2024-11-25T14:22:27,596 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732544577596
2024-11-25T14:22:27,598 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-25T14:22:27,598 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-25T14:22:27,599 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-25T14:22:27,599 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-25T14:22:27,603 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-25T14:22:27,603 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-25T14:22:27,604 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-25T14:22:27,604 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-25T14:22:27,605 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:27,605 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-25T14:22:27,608 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:27,616 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-25T14:22:27,617 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-25T14:22:27,618 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-25T14:22:27,620 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-25T14:22:27,620 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-25T14:22:27,622 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/cb69081f5976:0:becomeActiveMaster-HFileCleaner.large.0-1732544547621,5,FailOnTimeoutGroup]
2024-11-25T14:22:27,625 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/cb69081f5976:0:becomeActiveMaster-HFileCleaner.small.0-1732544547622,5,FailOnTimeoutGroup]
2024-11-25T14:22:27,625 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:27,625 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-25T14:22:27,626 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:27,627 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:27,633 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741831_1007 (size=1321)
2024-11-25T14:22:27,640 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34081, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-25T14:22:27,645 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44721 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cb69081f5976,36701,1732544544686
2024-11-25T14:22:27,648 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44721 {}] master.ServerManager(517): Registering regionserver=cb69081f5976,36701,1732544544686
2024-11-25T14:22:27,662 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581
2024-11-25T14:22:27,662 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38307
2024-11-25T14:22:27,663 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-25T14:22:27,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-25T14:22:27,677 DEBUG [RS:0;cb69081f5976:36701 {}] zookeeper.ZKUtil(111): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cb69081f5976,36701,1732544544686
2024-11-25T14:22:27,677 WARN  [RS:0;cb69081f5976:36701 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-25T14:22:27,677 INFO  [RS:0;cb69081f5976:36701 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-25T14:22:27,678 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/WALs/cb69081f5976,36701,1732544544686
2024-11-25T14:22:27,680 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cb69081f5976,36701,1732544544686]
2024-11-25T14:22:27,704 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-25T14:22:27,722 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-25T14:22:27,727 INFO  [RS:0;cb69081f5976:36701 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-25T14:22:27,727 INFO  [RS:0;cb69081f5976:36701 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:27,728 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-25T14:22:27,734 INFO  [RS:0;cb69081f5976:36701 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-25T14:22:27,736 INFO  [RS:0;cb69081f5976:36701 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:27,736 DEBUG [RS:0;cb69081f5976:36701 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cb69081f5976:0, corePoolSize=1, maxPoolSize=1
2024-11-25T14:22:27,737 DEBUG [RS:0;cb69081f5976:36701 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cb69081f5976:0, corePoolSize=1, maxPoolSize=1
2024-11-25T14:22:27,737 DEBUG [RS:0;cb69081f5976:36701 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0, corePoolSize=1, maxPoolSize=1
2024-11-25T14:22:27,737 DEBUG [RS:0;cb69081f5976:36701 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cb69081f5976:0, corePoolSize=1, maxPoolSize=1
2024-11-25T14:22:27,737 DEBUG [RS:0;cb69081f5976:36701 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cb69081f5976:0, corePoolSize=1, maxPoolSize=1
2024-11-25T14:22:27,738 DEBUG [RS:0;cb69081f5976:36701 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cb69081f5976:0, corePoolSize=2, maxPoolSize=2
2024-11-25T14:22:27,738 DEBUG [RS:0;cb69081f5976:36701 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cb69081f5976:0, corePoolSize=1, maxPoolSize=1
2024-11-25T14:22:27,738 DEBUG [RS:0;cb69081f5976:36701 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cb69081f5976:0, corePoolSize=1, maxPoolSize=1
2024-11-25T14:22:27,738 DEBUG [RS:0;cb69081f5976:36701 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cb69081f5976:0, corePoolSize=1, maxPoolSize=1
2024-11-25T14:22:27,738 DEBUG [RS:0;cb69081f5976:36701 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cb69081f5976:0, corePoolSize=1, maxPoolSize=1
2024-11-25T14:22:27,739 DEBUG [RS:0;cb69081f5976:36701 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cb69081f5976:0, corePoolSize=1, maxPoolSize=1
2024-11-25T14:22:27,739 DEBUG [RS:0;cb69081f5976:36701 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cb69081f5976:0, corePoolSize=1, maxPoolSize=1
2024-11-25T14:22:27,739 DEBUG [RS:0;cb69081f5976:36701 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cb69081f5976:0, corePoolSize=3, maxPoolSize=3
2024-11-25T14:22:27,739 DEBUG [RS:0;cb69081f5976:36701 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cb69081f5976:0, corePoolSize=3, maxPoolSize=3
2024-11-25T14:22:27,742 INFO  [RS:0;cb69081f5976:36701 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:27,742 INFO  [RS:0;cb69081f5976:36701 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:27,742 INFO  [RS:0;cb69081f5976:36701 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:27,742 INFO  [RS:0;cb69081f5976:36701 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:27,743 INFO  [RS:0;cb69081f5976:36701 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:27,743 INFO  [RS:0;cb69081f5976:36701 {}] hbase.ChoreService(168): Chore ScheduledChore name=cb69081f5976,36701,1732544544686-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-25T14:22:27,764 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-25T14:22:27,766 INFO  [RS:0;cb69081f5976:36701 {}] hbase.ChoreService(168): Chore ScheduledChore name=cb69081f5976,36701,1732544544686-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:27,766 INFO  [RS:0;cb69081f5976:36701 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:27,767 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.Replication(171): cb69081f5976,36701,1732544544686 started
2024-11-25T14:22:27,789 INFO  [RS:0;cb69081f5976:36701 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:27,789 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1482): Serving as cb69081f5976,36701,1732544544686, RpcServer on cb69081f5976/172.17.0.3:36701, sessionid=0x10173a96ace0001
2024-11-25T14:22:27,790 DEBUG [RS:0;cb69081f5976:36701 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-25T14:22:27,790 DEBUG [RS:0;cb69081f5976:36701 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cb69081f5976,36701,1732544544686
2024-11-25T14:22:27,790 DEBUG [RS:0;cb69081f5976:36701 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cb69081f5976,36701,1732544544686'
2024-11-25T14:22:27,790 DEBUG [RS:0;cb69081f5976:36701 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-25T14:22:27,791 DEBUG [RS:0;cb69081f5976:36701 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-25T14:22:27,792 DEBUG [RS:0;cb69081f5976:36701 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-25T14:22:27,792 DEBUG [RS:0;cb69081f5976:36701 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-25T14:22:27,792 DEBUG [RS:0;cb69081f5976:36701 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cb69081f5976,36701,1732544544686
2024-11-25T14:22:27,792 DEBUG [RS:0;cb69081f5976:36701 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cb69081f5976,36701,1732544544686'
2024-11-25T14:22:27,792 DEBUG [RS:0;cb69081f5976:36701 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-25T14:22:27,793 DEBUG [RS:0;cb69081f5976:36701 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-25T14:22:27,794 DEBUG [RS:0;cb69081f5976:36701 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-25T14:22:27,794 INFO  [RS:0;cb69081f5976:36701 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-25T14:22:27,794 INFO  [RS:0;cb69081f5976:36701 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-25T14:22:27,899 INFO  [RS:0;cb69081f5976:36701 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-11-25T14:22:27,903 INFO  [RS:0;cb69081f5976:36701 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cb69081f5976%2C36701%2C1732544544686, suffix=, logDir=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/WALs/cb69081f5976,36701,1732544544686, archiveDir=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/oldWALs, maxLogs=32
2024-11-25T14:22:27,925 DEBUG [RS:0;cb69081f5976:36701 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/WALs/cb69081f5976,36701,1732544544686/cb69081f5976%2C36701%2C1732544544686.1732544547907, exclude list is [], retry=0
2024-11-25T14:22:27,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37535,DS-1aab8e27-0a6e-4df2-9f24-0e782c02ea10,DISK]
2024-11-25T14:22:27,933 INFO  [RS:0;cb69081f5976:36701 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/WALs/cb69081f5976,36701,1732544544686/cb69081f5976%2C36701%2C1732544544686.1732544547907
2024-11-25T14:22:27,934 DEBUG [RS:0;cb69081f5976:36701 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45247:45247)]
2024-11-25T14:22:28,037 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-25T14:22:28,038 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581
2024-11-25T14:22:28,049 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741833_1009 (size=32)
2024-11-25T14:22:28,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-25T14:22:28,455 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-25T14:22:28,462 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-25T14:22:28,462 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:28,464 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-25T14:22:28,464 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-25T14:22:28,468 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-25T14:22:28,468 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:28,470 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-25T14:22:28,470 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-25T14:22:28,473 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-25T14:22:28,473 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:28,474 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-25T14:22:28,475 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-25T14:22:28,478 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-25T14:22:28,478 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:28,480 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-25T14:22:28,480 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-25T14:22:28,482 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740
2024-11-25T14:22:28,483 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740
2024-11-25T14:22:28,486 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-25T14:22:28,486 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-25T14:22:28,488 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-25T14:22:28,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-25T14:22:28,495 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-25T14:22:28,496 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59624745, jitterRate=-0.11152206361293793}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-25T14:22:28,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732544548452Initializing all the Stores at 1732544548454 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732544548455 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732544548455Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732544548455Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732544548455Cleaning up temporary data from old regions at 1732544548486 (+31 ms)Region opened successfully at 1732544548500 (+14 ms)
2024-11-25T14:22:28,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-25T14:22:28,501 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-25T14:22:28,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-25T14:22:28,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-25T14:22:28,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-25T14:22:28,503 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-25T14:22:28,504 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732544548501Disabling compacts and flushes for region at 1732544548501Disabling writes for close at 1732544548501Writing region close event to WAL at 1732544548503 (+2 ms)Closed at 1732544548503
2024-11-25T14:22:28,508 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-25T14:22:28,508 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-25T14:22:28,515 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-25T14:22:28,524 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-25T14:22:28,527 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-25T14:22:28,680 DEBUG [cb69081f5976:44721 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-25T14:22:28,692 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=cb69081f5976,36701,1732544544686
2024-11-25T14:22:28,698 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cb69081f5976,36701,1732544544686, state=OPENING
2024-11-25T14:22:28,709 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-25T14:22:28,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:22:28,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:22:28,719 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-25T14:22:28,719 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-25T14:22:28,721 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-25T14:22:28,723 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=cb69081f5976,36701,1732544544686}]
2024-11-25T14:22:28,899 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-25T14:22:28,903 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52797, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-25T14:22:28,915 INFO  [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-25T14:22:28,916 INFO  [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-25T14:22:28,916 INFO  [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta
2024-11-25T14:22:28,920 INFO  [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cb69081f5976%2C36701%2C1732544544686.meta, suffix=.meta, logDir=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/WALs/cb69081f5976,36701,1732544544686, archiveDir=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/oldWALs, maxLogs=32
2024-11-25T14:22:28,937 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/WALs/cb69081f5976,36701,1732544544686/cb69081f5976%2C36701%2C1732544544686.meta.1732544548922.meta, exclude list is [], retry=0
2024-11-25T14:22:28,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37535,DS-1aab8e27-0a6e-4df2-9f24-0e782c02ea10,DISK]
2024-11-25T14:22:28,946 INFO  [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/WALs/cb69081f5976,36701,1732544544686/cb69081f5976%2C36701%2C1732544544686.meta.1732544548922.meta
2024-11-25T14:22:28,947 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45247:45247)]
2024-11-25T14:22:28,947 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-25T14:22:28,949 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-25T14:22:28,953 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-25T14:22:28,960 INFO  [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-25T14:22:28,965 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-25T14:22:28,966 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-25T14:22:28,966 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-25T14:22:28,966 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-25T14:22:28,970 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-25T14:22:28,972 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-25T14:22:28,972 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:28,973 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-25T14:22:28,974 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-25T14:22:28,976 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-25T14:22:28,976 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:28,977 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-25T14:22:28,977 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-25T14:22:28,979 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-25T14:22:28,979 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:28,981 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-25T14:22:28,981 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-25T14:22:28,983 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-25T14:22:28,983 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:28,984 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-25T14:22:28,984 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-25T14:22:28,986 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740
2024-11-25T14:22:28,988 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740
2024-11-25T14:22:28,990 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-25T14:22:28,991 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-25T14:22:28,991 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-25T14:22:28,994 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-25T14:22:28,995 INFO  [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60767720, jitterRate=-0.0944904088973999}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-25T14:22:28,995 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-25T14:22:28,997 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732544548967Writing region info on filesystem at 1732544548967Initializing all the Stores at 1732544548969 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732544548969Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732544548970 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732544548970Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732544548970Cleaning up temporary data from old regions at 1732544548991 (+21 ms)Running coprocessor post-open hooks at 1732544548995 (+4 ms)Region opened successfully at 1732544548997 (+2 ms)
2024-11-25T14:22:29,003 INFO  [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732544548891
2024-11-25T14:22:29,012 DEBUG [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-25T14:22:29,013 INFO  [RS_OPEN_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-25T14:22:29,014 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=cb69081f5976,36701,1732544544686
2024-11-25T14:22:29,017 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cb69081f5976,36701,1732544544686, state=OPEN
2024-11-25T14:22:29,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-25T14:22:29,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-25T14:22:29,051 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-25T14:22:29,051 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-25T14:22:29,052 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=cb69081f5976,36701,1732544544686
2024-11-25T14:22:29,057 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-25T14:22:29,057 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=cb69081f5976,36701,1732544544686 in 329 msec
2024-11-25T14:22:29,063 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-25T14:22:29,063 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 545 msec
2024-11-25T14:22:29,065 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-25T14:22:29,065 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-25T14:22:29,088 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-25T14:22:29,090 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb69081f5976,36701,1732544544686, seqNum=-1]
2024-11-25T14:22:29,109 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-25T14:22:29,111 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43211, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-25T14:22:29,132 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6290 sec
2024-11-25T14:22:29,132 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732544549132, completionTime=-1
2024-11-25T14:22:29,136 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-25T14:22:29,136 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-25T14:22:29,167 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-25T14:22:29,167 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732544609167
2024-11-25T14:22:29,167 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732544669167
2024-11-25T14:22:29,167 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 31 msec
2024-11-25T14:22:29,170 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb69081f5976,44721,1732544543804-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:29,170 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb69081f5976,44721,1732544543804-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:29,170 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb69081f5976,44721,1732544543804-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:29,172 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-cb69081f5976:44721, period=300000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:29,172 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:29,173 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:29,178 DEBUG [master/cb69081f5976:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-25T14:22:29,211 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.300sec
2024-11-25T14:22:29,212 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-25T14:22:29,214 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-25T14:22:29,215 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-25T14:22:29,216 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-25T14:22:29,216 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-25T14:22:29,217 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb69081f5976,44721,1732544543804-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-25T14:22:29,217 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb69081f5976,44721,1732544543804-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-25T14:22:29,225 DEBUG [master/cb69081f5976:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-25T14:22:29,225 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-25T14:22:29,226 INFO  [master/cb69081f5976:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb69081f5976,44721,1732544543804-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-25T14:22:29,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@552bc2d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T14:22:29,244 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-25T14:22:29,244 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-25T14:22:29,247 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb69081f5976,44721,-1 for getting cluster id
2024-11-25T14:22:29,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-25T14:22:29,276 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b9c5d22a-5299-4650-a1d7-40bf093ed607'
2024-11-25T14:22:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-25T14:22:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b9c5d22a-5299-4650-a1d7-40bf093ed607"

2024-11-25T14:22:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@186cc2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T14:22:29,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb69081f5976,44721,-1]
2024-11-25T14:22:29,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-25T14:22:29,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T14:22:29,295 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-25T14:22:29,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@192f5cc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T14:22:29,298 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-25T14:22:29,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb69081f5976,36701,1732544544686, seqNum=-1]
2024-11-25T14:22:29,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-25T14:22:29,309 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-25T14:22:29,332 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=cb69081f5976,44721,1732544543804
2024-11-25T14:22:29,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@908c3e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T14:22:29,334 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb69081f5976,44721,-1 for getting cluster id
2024-11-25T14:22:29,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-25T14:22:29,336 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b9c5d22a-5299-4650-a1d7-40bf093ed607'
2024-11-25T14:22:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-25T14:22:29,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b9c5d22a-5299-4650-a1d7-40bf093ed607"

2024-11-25T14:22:29,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6078509c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T14:22:29,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb69081f5976,44721,-1]
2024-11-25T14:22:29,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-25T14:22:29,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T14:22:29,341 INFO  [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-25T14:22:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b461bcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T14:22:29,356 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=523, ProcessCount=11, AvailableMemoryMB=4763
2024-11-25T14:22:29,365 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-25T14:22:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is cb69081f5976,44721,1732544543804
2024-11-25T14:22:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@21b5079c
2024-11-25T14:22:29,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-25T14:22:29,376 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-25T14:22:29,377 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-25T14:22:29,393 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:29,398 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7e0b9dbe to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@527142be
2024-11-25T14:22:29,399 WARN  [PEWorker-3 {}] client.ZKConnectionRegistry(87): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry
2024-11-25T14:22:29,439 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x7e0b9dbe {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7e0b9dbe to 127.0.0.1:51135
2024-11-25T14:22:29,439 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:29,441 INFO  [PEWorker-3 {}] master.HMaster(2490): Client=null/null create 'hbase:replication', {TABLE_ATTRIBUTES => {coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.regionserver.region.split_restriction.delimiter' => '-', 'hbase.regionserver.region.split_restriction.type' => 'DelimitedKeyPrefix'}}}, {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-25T14:22:29,447 DEBUG [PEWorker-3 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:replication
2024-11-25T14:22:29,450 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_PRE_OPERATION
2024-11-25T14:22:29,451 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:29,454 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-25T14:22:29,470 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741835_1011 (size=1138)
2024-11-25T14:22:29,499 DEBUG [PEWorker-3 {}] procedure.ProcedureSyncWait(219): waitFor Creating table hbase:replication
2024-11-25T14:22:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-25T14:22:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-25T14:22:29,874 INFO  [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c4a88aaabbd3b8251330812a470ab6d5, NAME => 'hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:replication', {TABLE_ATTRIBUTES => {coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.regionserver.region.split_restriction.delimiter' => '-', 'hbase.regionserver.region.split_restriction.type' => 'DelimitedKeyPrefix', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581
2024-11-25T14:22:29,886 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741836_1012 (size=44)
2024-11-25T14:22:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-25T14:22:30,288 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-25T14:22:30,288 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1722): Closing c4a88aaabbd3b8251330812a470ab6d5, disabling compactions & flushes
2024-11-25T14:22:30,289 INFO  [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.
2024-11-25T14:22:30,289 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.
2024-11-25T14:22:30,289 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5. after waiting 0 ms
2024-11-25T14:22:30,289 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.
2024-11-25T14:22:30,289 INFO  [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1973): Closed hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.
2024-11-25T14:22:30,289 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1676): Region close journal for c4a88aaabbd3b8251330812a470ab6d5:
Waiting for close lock at 1732544550288Disabling compacts and flushes for region at 1732544550288Disabling writes for close at 1732544550289 (+1 ms)Writing region close event to WAL at 1732544550289Closed at 1732544550289
2024-11-25T14:22:30,293 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_ADD_TO_META
2024-11-25T14:22:30,301 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.","families":{"info":[{"qualifier":"regioninfo","vlen":43,"tag":[],"timestamp":"1732544550294"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732544550294"}]},"ts":"1732544550294"}
2024-11-25T14:22:30,309 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-25T14:22:30,312 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-25T14:22:30,314 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:replication","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732544550312"}]},"ts":"1732544550312"}
2024-11-25T14:22:30,320 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:replication, state=ENABLING in hbase:meta
2024-11-25T14:22:30,322 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=c4a88aaabbd3b8251330812a470ab6d5, ASSIGN}]
2024-11-25T14:22:30,325 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=c4a88aaabbd3b8251330812a470ab6d5, ASSIGN
2024-11-25T14:22:30,328 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:replication, region=c4a88aaabbd3b8251330812a470ab6d5, ASSIGN; state=OFFLINE, location=cb69081f5976,36701,1732544544686; forceNewPlan=false, retain=false
2024-11-25T14:22:30,479 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=c4a88aaabbd3b8251330812a470ab6d5, regionState=OPENING, regionLocation=cb69081f5976,36701,1732544544686
2024-11-25T14:22:30,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:replication, region=c4a88aaabbd3b8251330812a470ab6d5, ASSIGN because future has completed
2024-11-25T14:22:30,487 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=7, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4a88aaabbd3b8251330812a470ab6d5, server=cb69081f5976,36701,1732544544686}]
2024-11-25T14:22:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-25T14:22:30,650 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] handler.AssignRegionHandler(132): Open hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.
2024-11-25T14:22:30,650 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-25T14:22:30,650 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] monitor.StreamSlowMonitor(122): New stream slow monitor rep
2024-11-25T14:22:30,654 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cb69081f5976%2C36701%2C1732544544686.rep, suffix=, logDir=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/WALs/cb69081f5976,36701,1732544544686, archiveDir=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/oldWALs, maxLogs=32
2024-11-25T14:22:30,670 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/WALs/cb69081f5976,36701,1732544544686/cb69081f5976%2C36701%2C1732544544686.rep.1732544550656, exclude list is [], retry=0
2024-11-25T14:22:30,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37535,DS-1aab8e27-0a6e-4df2-9f24-0e782c02ea10,DISK]
2024-11-25T14:22:30,680 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/WALs/cb69081f5976,36701,1732544544686/cb69081f5976%2C36701%2C1732544544686.rep.1732544550656
2024-11-25T14:22:30,682 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45247:45247)]
2024-11-25T14:22:30,683 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7752): Opening region: {ENCODED => c4a88aaabbd3b8251330812a470ab6d5, NAME => 'hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.', STARTKEY => '', ENDKEY => ''}
2024-11-25T14:22:30,684 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-25T14:22:30,684 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5. service=MultiRowMutationService
2024-11-25T14:22:30,684 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:replication successfully.
2024-11-25T14:22:30,684 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table replication c4a88aaabbd3b8251330812a470ab6d5
2024-11-25T14:22:30,685 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(898): Instantiated hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-25T14:22:30,685 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7794): checking encryption for c4a88aaabbd3b8251330812a470ab6d5
2024-11-25T14:22:30,685 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7797): checking classloading for c4a88aaabbd3b8251330812a470ab6d5
2024-11-25T14:22:30,689 INFO  [StoreOpener-c4a88aaabbd3b8251330812a470ab6d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family hfileref of region c4a88aaabbd3b8251330812a470ab6d5 
2024-11-25T14:22:30,692 INFO  [StoreOpener-c4a88aaabbd3b8251330812a470ab6d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4a88aaabbd3b8251330812a470ab6d5 columnFamilyName hfileref
2024-11-25T14:22:30,693 DEBUG [StoreOpener-c4a88aaabbd3b8251330812a470ab6d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:30,694 INFO  [StoreOpener-c4a88aaabbd3b8251330812a470ab6d5-1 {}] regionserver.HStore(327): Store=c4a88aaabbd3b8251330812a470ab6d5/hfileref,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-25T14:22:30,695 INFO  [StoreOpener-c4a88aaabbd3b8251330812a470ab6d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family queue of region c4a88aaabbd3b8251330812a470ab6d5 
2024-11-25T14:22:30,698 INFO  [StoreOpener-c4a88aaabbd3b8251330812a470ab6d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4a88aaabbd3b8251330812a470ab6d5 columnFamilyName queue
2024-11-25T14:22:30,698 DEBUG [StoreOpener-c4a88aaabbd3b8251330812a470ab6d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:30,699 INFO  [StoreOpener-c4a88aaabbd3b8251330812a470ab6d5-1 {}] regionserver.HStore(327): Store=c4a88aaabbd3b8251330812a470ab6d5/queue,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-25T14:22:30,699 INFO  [StoreOpener-c4a88aaabbd3b8251330812a470ab6d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family sid of region c4a88aaabbd3b8251330812a470ab6d5 
2024-11-25T14:22:30,705 INFO  [StoreOpener-c4a88aaabbd3b8251330812a470ab6d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4a88aaabbd3b8251330812a470ab6d5 columnFamilyName sid
2024-11-25T14:22:30,705 DEBUG [StoreOpener-c4a88aaabbd3b8251330812a470ab6d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-25T14:22:30,706 INFO  [StoreOpener-c4a88aaabbd3b8251330812a470ab6d5-1 {}] regionserver.HStore(327): Store=c4a88aaabbd3b8251330812a470ab6d5/sid,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-25T14:22:30,707 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1038): replaying wal for c4a88aaabbd3b8251330812a470ab6d5
2024-11-25T14:22:30,709 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/replication/c4a88aaabbd3b8251330812a470ab6d5
2024-11-25T14:22:30,710 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/replication/c4a88aaabbd3b8251330812a470ab6d5
2024-11-25T14:22:30,713 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1048): stopping wal replay for c4a88aaabbd3b8251330812a470ab6d5
2024-11-25T14:22:30,713 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1060): Cleaning up temporary data for c4a88aaabbd3b8251330812a470ab6d5
2024-11-25T14:22:30,715 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:replication descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead.
2024-11-25T14:22:30,719 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1093): writing seq id for c4a88aaabbd3b8251330812a470ab6d5
2024-11-25T14:22:30,729 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/replication/c4a88aaabbd3b8251330812a470ab6d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-25T14:22:30,730 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1114): Opened c4a88aaabbd3b8251330812a470ab6d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67890464, jitterRate=0.011646747589111328}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242}
2024-11-25T14:22:30,731 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c4a88aaabbd3b8251330812a470ab6d5
2024-11-25T14:22:30,732 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1006): Region open journal for c4a88aaabbd3b8251330812a470ab6d5:
Running coprocessor pre-open hook at 1732544550685Writing region info on filesystem at 1732544550685Initializing all the Stores at 1732544550688 (+3 ms)Instantiating store for column family {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732544550688Instantiating store for column family {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732544550688Instantiating store for column family {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732544550689 (+1 ms)Cleaning up temporary data from old regions at 1732544550713 (+24 ms)Running coprocessor post-open hooks at 1732544550731 (+18 ms)Region opened successfully at 1732544550732 (+1 ms)
2024-11-25T14:22:30,734 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5., pid=7, masterSystemTime=1732544550643
2024-11-25T14:22:30,738 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.
2024-11-25T14:22:30,739 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] handler.AssignRegionHandler(153): Opened hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.
2024-11-25T14:22:30,741 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=c4a88aaabbd3b8251330812a470ab6d5, regionState=OPEN, openSeqNum=2, regionLocation=cb69081f5976,36701,1732544544686
2024-11-25T14:22:30,747 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4a88aaabbd3b8251330812a470ab6d5, server=cb69081f5976,36701,1732544544686 because future has completed
2024-11-25T14:22:30,760 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=7, resume processing ppid=6
2024-11-25T14:22:30,760 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure c4a88aaabbd3b8251330812a470ab6d5, server=cb69081f5976,36701,1732544544686 in 264 msec
2024-11-25T14:22:30,768 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-25T14:22:30,768 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=c4a88aaabbd3b8251330812a470ab6d5, ASSIGN in 439 msec
2024-11-25T14:22:30,771 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-25T14:22:30,772 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:replication","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732544550771"}]},"ts":"1732544550771"}
2024-11-25T14:22:30,777 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:replication, state=ENABLED in hbase:meta
2024-11-25T14:22:30,780 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_POST_OPERATION
2024-11-25T14:22:30,788 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:replication in 1.3380 sec
2024-11-25T14:22:30,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='1-', locateType=CURRENT is [region=hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5., hostname=cb69081f5976,36701,1732544544686, seqNum=2]
2024-11-25T14:22:30,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms
2024-11-25T14:22:30,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:22:30,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:22:30,898 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=4, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:31,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=8
2024-11-25T14:22:31,066 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:22:31,134 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:31,136 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=8}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-25T14:22:31,137 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=8}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb69081f5976,36701,1732544544686, seqNum=-1]
2024-11-25T14:22:31,137 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=8}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-25T14:22:31,139 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49533, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService
2024-11-25T14:22:31,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='1-cb69081f5976,36701,1732544544686', locateType=CURRENT is [region=hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5., hostname=cb69081f5976,36701,1732544544686, seqNum=2]
2024-11-25T14:22:31,150 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-25T14:22:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-25T14:22:31,157 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:31,162 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=4
2024-11-25T14:22:31,162 INFO  [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:22:31,162 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 259 msec
2024-11-25T14:22:31,166 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 1.7820 sec
2024-11-25T14:22:31,172 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:31,176 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1150014d to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7f435c7d
2024-11-25T14:22:31,185 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:31,186 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-25T14:22:31,543 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:31,547 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-25T14:22:31,556 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:31,559 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7f06493c to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@556d90d
2024-11-25T14:22:31,568 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x7f06493c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7f06493c to 127.0.0.1:51135
2024-11-25T14:22:31,569 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:31,586 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-25T14:22:31,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=10
2024-11-25T14:22:31,743 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:31,745 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:31,745 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:31,745 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:31,745 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:31,749 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x104c955a to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2e427f78
2024-11-25T14:22:31,750 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:31,772 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:31,772 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:31,773 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-25T14:22:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-25T14:22:31,777 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:31,781 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-25T14:22:31,781 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:31,781 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec
2024-11-25T14:22:31,784 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 234 msec
2024-11-25T14:22:31,791 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:31,793 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x525ec58d to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2b1e570
2024-11-25T14:22:31,801 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:31,802 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:31,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-25T14:22:31,873 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:31,878 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:31,880 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:31,881 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:31,885 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false
2024-11-25T14:22:31,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:31,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-25T14:22:31,889 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:31,892 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3939ecfa to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2fe3fd9f
2024-11-25T14:22:31,901 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x3939ecfa {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3939ecfa to 127.0.0.1:51135
2024-11-25T14:22:31,901 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:31,920 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:31,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-25T14:22:32,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=12
2024-11-25T14:22:32,075 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:32,077 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:32,077 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:32,077 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:32,077 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:32,079 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x52113e9a to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5e076d2a
2024-11-25T14:22:32,080 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:32,103 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:32,103 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:32,104 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-25T14:22:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-25T14:22:32,108 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:32,113 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-25T14:22:32,113 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec
2024-11-25T14:22:32,114 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false
2024-11-25T14:22:32,118 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 229 msec
2024-11-25T14:22:32,122 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:32,124 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x73a8d8bd to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@688185cd
2024-11-25T14:22:32,135 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:32,135 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-25T14:22:32,203 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:32,204 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:32,205 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:32,207 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:32,208 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:32,210 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false
2024-11-25T14:22:32,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-25T14:22:32,214 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:32,216 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4f7e14fd to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1e48c34
2024-11-25T14:22:32,226 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x4f7e14fd {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4f7e14fd to 127.0.0.1:51135
2024-11-25T14:22:32,226 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:32,244 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:32,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-25T14:22:32,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=14
2024-11-25T14:22:32,400 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:32,402 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:32,402 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:32,402 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:32,402 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:32,406 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x46589f87 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@327b893a
2024-11-25T14:22:32,407 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:32,430 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:32,431 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:32,431 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-25T14:22:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-25T14:22:32,435 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:32,440 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-25T14:22:32,440 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false
2024-11-25T14:22:32,440 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec
2024-11-25T14:22:32,443 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 230 msec
2024-11-25T14:22:32,449 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:32,452 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x154b13f8 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@41307365
2024-11-25T14:22:32,459 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:32,460 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:32,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-25T14:22:32,533 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:32,534 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:32,536 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:32,538 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:32,539 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15
2024-11-25T14:22:32,543 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:32,545 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7830488e to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4bbb1552
2024-11-25T14:22:32,551 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x7830488e {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7830488e to 127.0.0.1:51135
2024-11-25T14:22:32,551 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:32,569 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15
2024-11-25T14:22:32,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=16
2024-11-25T14:22:32,725 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:32,726 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:32,726 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:32,726 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:32,727 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:32,729 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x61f4d16f to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@79ea6a68
2024-11-25T14:22:32,729 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:32,749 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:32,749 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:32,749 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16
2024-11-25T14:22:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=16
2024-11-25T14:22:32,753 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:32,758 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15
2024-11-25T14:22:32,758 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 184 msec
2024-11-25T14:22:32,758 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:32,762 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 219 msec
2024-11-25T14:22:32,769 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:32,773 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x60ffcb8b to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@16bfdc18
2024-11-25T14:22:32,785 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:32,786 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15
2024-11-25T14:22:32,862 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:32,863 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:32,864 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:32,865 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false
2024-11-25T14:22:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17
2024-11-25T14:22:32,870 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:32,872 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58a2e5c5 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@233c24d5
2024-11-25T14:22:32,885 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x58a2e5c5 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x58a2e5c5 to 127.0.0.1:51135
2024-11-25T14:22:32,885 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:32,902 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17
2024-11-25T14:22:33,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=18
2024-11-25T14:22:33,056 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:33,058 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:33,058 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:33,058 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:33,058 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:33,060 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x71ff003a to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5b44ddd7
2024-11-25T14:22:33,060 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:33,078 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:33,078 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:33,078 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18
2024-11-25T14:22:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=18
2024-11-25T14:22:33,082 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:33,086 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17
2024-11-25T14:22:33,086 INFO  [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false
2024-11-25T14:22:33,086 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 180 msec
2024-11-25T14:22:33,088 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 221 msec
2024-11-25T14:22:33,095 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:33,096 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4e224bfa to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@643a51fd
2024-11-25T14:22:33,109 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:33,109 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:33,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17
2024-11-25T14:22:33,182 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:33,183 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:33,184 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:33,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19
2024-11-25T14:22:33,187 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:33,188 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4a3da167 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@62930a49
2024-11-25T14:22:33,200 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x4a3da167 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4a3da167 to 127.0.0.1:51135
2024-11-25T14:22:33,200 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:33,219 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:33,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19
2024-11-25T14:22:33,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=20
2024-11-25T14:22:33,373 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:33,374 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:33,374 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:33,375 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:33,375 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:33,376 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x659a0864 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@746ed8f8
2024-11-25T14:22:33,377 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:33,397 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:33,397 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:33,397 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20
2024-11-25T14:22:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=20
2024-11-25T14:22:33,402 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:33,405 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19
2024-11-25T14:22:33,405 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:33,405 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec
2024-11-25T14:22:33,408 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 221 msec
2024-11-25T14:22:33,414 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:33,415 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x19a3e3cf to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6249c05
2024-11-25T14:22:33,426 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:33,427 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19
2024-11-25T14:22:33,502 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:33,503 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:33,507 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:33,507 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22
2024-11-25T14:22:33,515 DEBUG [PEWorker-2 {}] procedure.MasterProcedureScheduler(648): PEER '1', shared lock count=0, exclusively locked by procId=21
2024-11-25T14:22:33,515 DEBUG [PEWorker-2 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=22, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21
2024-11-25T14:22:33,518 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:33,518 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:33,519 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:33,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:33,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:33,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:33,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:33,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:33,529 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22
2024-11-25T14:22:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21
2024-11-25T14:22:33,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=23
2024-11-25T14:22:33,683 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:22:33,683 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:22:33,683 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:22:33,684 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:33,684 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:33,685 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0293e24f to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3b93f405
2024-11-25T14:22:33,686 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:33,686 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:22:33,691 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=23
2024-11-25T14:22:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=23
2024-11-25T14:22:33,694 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:33,697 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=21
2024-11-25T14:22:33,698 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:22:33,698 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 166 msec
2024-11-25T14:22:33,699 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:22:33,733 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:replication'
2024-11-25T14:22:33,736 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-25T14:22:33,742 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:22:33,745 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 235 msec
2024-11-25T14:22:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22
2024-11-25T14:22:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21
2024-11-25T14:22:33,833 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:33,834 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24
2024-11-25T14:22:33,837 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:33,844 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=24, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:33,847 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=24, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=11 msec
2024-11-25T14:22:33,847 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24
2024-11-25T14:22:33,944 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='', locateType=CURRENT is [region=hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5., hostname=cb69081f5976,36701,1732544544686, seqNum=2]
2024-11-25T14:22:33,995 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=266 (was 214)
Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-965633759_22 at /127.0.0.1:52592 [Cleaning up]
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
	app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
	java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
	java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263)
	java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334)
	java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72)
	app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x46589f87
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1150014d-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1150014d-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x525ec58d-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x46589f87-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ForkJoinPool.commonPool-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x60ffcb8b-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x104c955a
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x60ffcb8b
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RSProcedureDispatcher-pool-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x52113e9a-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x104c955a-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS_REFRESH_PEER-regionserver/cb69081f5976:0-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0293e24f-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x61f4d16f-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x659a0864-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x52113e9a
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x659a0864
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x525ec58d
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x19a3e3cf-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0293e24f
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0293e24f-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x73a8d8bd
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4e224bfa-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x52113e9a-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/cb69081f5976:0-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x19a3e3cf-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x61f4d16f-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x60ffcb8b-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x154b13f8-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x154b13f8-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x19a3e3cf
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x73a8d8bd-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: PacketResponder: BP-610877534-172.17.0.3-1732544540075:blk_1073741837_1013, type=LAST_IN_PIPELINE
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367)
	app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x154b13f8
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x61f4d16f
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x71ff003a-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x73a8d8bd-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x525ec58d-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: RSProcedureDispatcher-pool-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581-prefix:cb69081f5976,36701,1732544544686.rep
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-965633759_22 at /127.0.0.1:52716 [Receiving block BP-610877534-172.17.0.3-1732544540075:blk_1073741837_1013]
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
	app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
	java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
	java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
	java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343)
	java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151)
	app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119)
	app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553)
	app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011)
	app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110)
	app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x71ff003a
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x104c955a-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x46589f87-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x71ff003a-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4e224bfa-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1150014d
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x659a0864-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4e224bfa
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=507 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=505 (was 523), ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=4695 (was 4763)
2024-11-25T14:22:34,006 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=265, OpenFileDescriptor=507, MaxFileDescriptor=1048576, SystemLoadAverage=505, ProcessCount=11, AvailableMemoryMB=4693
2024-11-25T14:22:34,008 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25
2024-11-25T14:22:34,014 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:34,016 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7d066b52 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@46518848
2024-11-25T14:22:34,026 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x7d066b52 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7d066b52 to 127.0.0.1:51135
2024-11-25T14:22:34,027 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:34,087 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:34,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25
2024-11-25T14:22:34,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22
2024-11-25T14:22:34,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=26
2024-11-25T14:22:34,242 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:22:34,278 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:34,282 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26
2024-11-25T14:22:34,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=26
2024-11-25T14:22:34,286 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:34,289 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25
2024-11-25T14:22:34,289 INFO  [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:22:34,289 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 200 msec
2024-11-25T14:22:34,291 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 281 msec
2024-11-25T14:22:34,299 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:34,300 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x111a8d23 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@259c964e
2024-11-25T14:22:34,310 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:34,310 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:34,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25
2024-11-25T14:22:34,333 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:34,334 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27
2024-11-25T14:22:34,339 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:34,341 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x558ce80d to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@16d3a017
2024-11-25T14:22:34,352 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x558ce80d {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x558ce80d to 127.0.0.1:51135
2024-11-25T14:22:34,352 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:34,369 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27
2024-11-25T14:22:34,477 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-25T14:22:34,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=28
2024-11-25T14:22:34,524 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:34,526 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:34,526 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:34,527 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:34,527 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:34,529 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1dcc87c8 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@17948616
2024-11-25T14:22:34,534 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:34,543 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-25T14:22:34,543 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-25T14:22:34,547 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_replication
2024-11-25T14:22:34,547 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_replication Metrics about Tables on a single HBase RegionServer
2024-11-25T14:22:34,548 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-25T14:22:34,548 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-25T14:22:34,549 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-25T14:22:34,549 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-25T14:22:34,560 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:34,560 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:34,561 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28
2024-11-25T14:22:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=28
2024-11-25T14:22:34,567 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:34,572 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27
2024-11-25T14:22:34,573 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:34,573 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 199 msec
2024-11-25T14:22:34,578 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 239 msec
2024-11-25T14:22:34,578 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:34,580 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4186676e to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1cf2d794
2024-11-25T14:22:34,593 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:34,593 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22
2024-11-25T14:22:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27
2024-11-25T14:22:34,655 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:34,658 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-11-25T14:22:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29
2024-11-25T14:22:34,663 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:34,664 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d05d9c8 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@57685da1
2024-11-25T14:22:34,685 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x0d05d9c8 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0d05d9c8 to 127.0.0.1:51135
2024-11-25T14:22:34,685 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:34,703 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29
2024-11-25T14:22:34,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=30
2024-11-25T14:22:34,858 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:34,860 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:34,860 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:34,860 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:34,860 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:34,862 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1718d672 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@477415ed
2024-11-25T14:22:34,862 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:34,888 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:34,888 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:34,889 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30
2024-11-25T14:22:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=30
2024-11-25T14:22:34,895 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:34,900 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29
2024-11-25T14:22:34,900 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-11-25T14:22:34,900 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 192 msec
2024-11-25T14:22:34,904 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 242 msec
2024-11-25T14:22:34,910 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:34,913 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0604e5c4 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@58e1d4cd
2024-11-25T14:22:34,926 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:34,927 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29
2024-11-25T14:22:34,983 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:34,984 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false
2024-11-25T14:22:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31
2024-11-25T14:22:34,990 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:34,991 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x06ea354c to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@67ef3e7d
2024-11-25T14:22:35,001 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x06ea354c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x06ea354c to 127.0.0.1:51135
2024-11-25T14:22:35,001 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:35,002 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:35,004 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=31, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:35,006 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=31, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=19 msec
2024-11-25T14:22:35,006 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31
2024-11-25T14:22:35,093 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
2024-11-25T14:22:35,094 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false
2024-11-25T14:22:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32
2024-11-25T14:22:35,098 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:35,100 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0fccffc9 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55113c25
2024-11-25T14:22:35,112 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x0fccffc9 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0fccffc9 to 127.0.0.1:51135
2024-11-25T14:22:35,112 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:35,128 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32
2024-11-25T14:22:35,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=33
2024-11-25T14:22:35,282 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:35,284 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:35,284 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:35,284 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:35,284 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:35,286 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4c715f1c to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@331f1425
2024-11-25T14:22:35,287 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:35,307 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:35,307 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:35,308 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33
2024-11-25T14:22:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=33
2024-11-25T14:22:35,312 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:35,315 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32
2024-11-25T14:22:35,315 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false
2024-11-25T14:22:35,315 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 185 msec
2024-11-25T14:22:35,318 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 221 msec
2024-11-25T14:22:35,326 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:35,328 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1fd0570b to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@23c4e691
2024-11-25T14:22:35,334 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:35,335 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32
2024-11-25T14:22:35,412 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:35,413 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false
2024-11-25T14:22:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34
2024-11-25T14:22:35,418 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:35,419 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b64f48e to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4fbad2b3
2024-11-25T14:22:35,426 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x2b64f48e {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2b64f48e to 127.0.0.1:51135
2024-11-25T14:22:35,426 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:35,427 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:35,429 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=34, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:35,432 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=34, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=17 msec
2024-11-25T14:22:35,432 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34
2024-11-25T14:22:35,523 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
2024-11-25T14:22:35,523 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35
2024-11-25T14:22:35,529 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:35,529 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:35,530 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:35,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:35,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:35,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:35,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:35,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:35,544 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35
2024-11-25T14:22:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22
2024-11-25T14:22:35,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=36
2024-11-25T14:22:35,698 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:22:35,698 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:22:35,698 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:22:35,699 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:35,699 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:35,700 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4d71d4c4 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4e3369ad
2024-11-25T14:22:35,700 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:35,700 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:22:35,703 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=36
2024-11-25T14:22:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=36
2024-11-25T14:22:35,706 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:35,709 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35
2024-11-25T14:22:35,709 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:22:35,709 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 162 msec
2024-11-25T14:22:35,709 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:22:35,716 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:22:35,718 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 192 msec
2024-11-25T14:22:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35
2024-11-25T14:22:35,842 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:35,843 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37
2024-11-25T14:22:35,845 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:35,846 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=37, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:35,848 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=37, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:22:35,848 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37
2024-11-25T14:22:35,953 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:22:35,954 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38
2024-11-25T14:22:35,956 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:35,958 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=38, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:35,959 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=38, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:22:35,959 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38
2024-11-25T14:22:36,063 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:36,078 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=290 (was 265)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4d71d4c4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1dcc87c8
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1dcc87c8-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4186676e-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4c715f1c
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4186676e-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4c715f1c-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1718d672-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4c715f1c-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x111a8d23
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0604e5c4-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0604e5c4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1fd0570b-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4d71d4c4-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4d71d4c4-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4186676e
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1718d672-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x111a8d23-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x111a8d23-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1718d672
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1dcc87c8-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1fd0570b
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1fd0570b-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0604e5c4-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
 - Thread LEAK? -, OpenFileDescriptor=540 (was 507) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=505 (was 505), ProcessCount=11 (was 11), AvailableMemoryMB=4644 (was 4693)
2024-11-25T14:22:36,087 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=290, OpenFileDescriptor=540, MaxFileDescriptor=1048576, SystemLoadAverage=505, ProcessCount=11, AvailableMemoryMB=4644
2024-11-25T14:22:36,089 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39
2024-11-25T14:22:36,092 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:36,094 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x559ed15d to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4a5bfa92
2024-11-25T14:22:36,134 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x559ed15d {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x559ed15d to 127.0.0.1:51135
2024-11-25T14:22:36,134 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:36,160 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39
2024-11-25T14:22:36,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=40
2024-11-25T14:22:36,314 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:22:36,342 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:36,345 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40
2024-11-25T14:22:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=40
2024-11-25T14:22:36,349 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:36,353 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39
2024-11-25T14:22:36,353 INFO  [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:22:36,353 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec
2024-11-25T14:22:36,356 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 264 msec
2024-11-25T14:22:36,365 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:36,366 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5e787cda to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4685a010
2024-11-25T14:22:36,376 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:36,376 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39
2024-11-25T14:22:36,412 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:36,413 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41
2024-11-25T14:22:36,418 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:36,419 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x42370bf2 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5bea9386
2024-11-25T14:22:36,426 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x42370bf2 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x42370bf2 to 127.0.0.1:51135
2024-11-25T14:22:36,426 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:36,427 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:36,429 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=41, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:36,430 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=41, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=16 msec
2024-11-25T14:22:36,430 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41
2024-11-25T14:22:36,523 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists
2024-11-25T14:22:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:22:36,531 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42
2024-11-25T14:22:36,534 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:36,537 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=42, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:36,539 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=42, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec
2024-11-25T14:22:36,539 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42
2024-11-25T14:22:36,643 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:22:36,645 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=43, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43
2024-11-25T14:22:36,648 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test2'
2024-11-25T14:22:36,649 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2655bbbf to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@54c28bf4
2024-11-25T14:22:36,668 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x2655bbbf {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2655bbbf to 127.0.0.1:51135
2024-11-25T14:22:36,668 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test2
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:36,718 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43
2024-11-25T14:22:36,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=44
2024-11-25T14:22:36,872 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=ADD_PEER
2024-11-25T14:22:36,901 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.ReplicationSource(231): queueId=2-cb69081f5976,36701,1732544544686, ReplicationSource: 2, currentBandwidth=0
2024-11-25T14:22:36,904 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=44
2024-11-25T14:22:36,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=44
2024-11-25T14:22:36,907 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:36,911 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43
2024-11-25T14:22:36,911 INFO  [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 2, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:22:36,911 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec
2024-11-25T14:22:36,913 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 266 msec
2024-11-25T14:22:36,921 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test2'
2024-11-25T14:22:36,922 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a5f3ad4 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7b2dc422
2024-11-25T14:22:36,943 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:36,943 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43
2024-11-25T14:22:36,962 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 2 completed
2024-11-25T14:22:36,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:22:36,963 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=45, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:36,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45
2024-11-25T14:22:36,968 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:36,968 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:36,969 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:36,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:36,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:36,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:36,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:36,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:36,977 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45
2024-11-25T14:22:37,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=46
2024-11-25T14:22:37,131 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:22:37,131 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:22:37,131 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:22:37,132 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:37,132 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:37,133 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x514f56e1 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8fc0413
2024-11-25T14:22:37,134 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:37,134 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:22:37,138 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=46
2024-11-25T14:22:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=46
2024-11-25T14:22:37,141 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:37,145 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=45
2024-11-25T14:22:37,145 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=45, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 164 msec
2024-11-25T14:22:37,145 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:22:37,145 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:22:37,154 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:22:37,156 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 191 msec
2024-11-25T14:22:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45
2024-11-25T14:22:37,282 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:22:37,284 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47
2024-11-25T14:22:37,289 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/sync-rep-state
2024-11-25T14:22:37,289 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state
2024-11-25T14:22:37,290 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/new-sync-rep-state
2024-11-25T14:22:37,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/new-sync-rep-state
2024-11-25T14:22:37,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2
2024-11-25T14:22:37,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state
2024-11-25T14:22:37,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/sync-rep-state
2024-11-25T14:22:37,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2
2024-11-25T14:22:37,302 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47
2024-11-25T14:22:37,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=48
2024-11-25T14:22:37,456 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=REMOVE_PEER
2024-11-25T14:22:37,456 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 2: 0
2024-11-25T14:22:37,456 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSource(687): peerId=2, Closing source 2-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:22:37,456 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=2, Interrupted while sleeping between retries
2024-11-25T14:22:37,457 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test2'
2024-11-25T14:22:37,458 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x06fcc4df to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7733033
2024-11-25T14:22:37,458 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:37,459 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(562): Done with the queue 2-cb69081f5976,36701,1732544544686
2024-11-25T14:22:37,461 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48
2024-11-25T14:22:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=48
2024-11-25T14:22:37,464 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:37,467 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47
2024-11-25T14:22:37,467 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 162 msec
2024-11-25T14:22:37,467 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 2, move on
2024-11-25T14:22:37,467 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 2, move on
2024-11-25T14:22:37,474 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 2
2024-11-25T14:22:37,476 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 190 msec
2024-11-25T14:22:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47
2024-11-25T14:22:37,603 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed
2024-11-25T14:22:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:22:37,605 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49
2024-11-25T14:22:37,609 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:37,612 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=49, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:37,614 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=49, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec
2024-11-25T14:22:37,614 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22
2024-11-25T14:22:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49
2024-11-25T14:22:37,713 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:22:37,713 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50
2024-11-25T14:22:37,716 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:37,719 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=50, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:37,720 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=50, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec
2024-11-25T14:22:37,721 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50
2024-11-25T14:22:37,823 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:37,841 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=302 (was 290)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x06fcc4df
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0a5f3ad4-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x5e787cda-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x06fcc4df-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0a5f3ad4-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x514f56e1-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0a5f3ad4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x5e787cda-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x5e787cda
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x06fcc4df-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x514f56e1-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x514f56e1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=555 (was 540) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=472 (was 505), ProcessCount=11 (was 11), AvailableMemoryMB=4617 (was 4644)
2024-11-25T14:22:37,851 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=302, OpenFileDescriptor=555, MaxFileDescriptor=1048576, SystemLoadAverage=472, ProcessCount=11, AvailableMemoryMB=4616
2024-11-25T14:22:37,853 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=51, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51
2024-11-25T14:22:37,856 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: java.lang.ClassNotFoundException: whatever
	at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?]
	at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?]
	at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?]
	at java.lang.Class.forName0(Native Method) ~[?:?]
	at java.lang.Class.forName(Class.java:375) ~[?:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?]
	... 11 more
2024-11-25T14:22:37,858 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=51, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
Caused by: java.lang.ClassNotFoundException: whatever
	at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?]
	at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?]
	at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?]
	at java.lang.Class.forName0(Native Method) ~[?:?]
	at java.lang.Class.forName(Class.java:375) ~[?:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:37,860 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=51, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=5 msec
2024-11-25T14:22:37,860 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51
2024-11-25T14:22:37,963 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever
2024-11-25T14:22:37,967 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52
2024-11-25T14:22:37,970 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:37,971 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=52, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:37,972 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=52, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:22:37,972 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52
2024-11-25T14:22:38,083 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:22:38,084 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53
2024-11-25T14:22:38,087 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:38,088 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=53, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:38,090 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=53, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec
2024-11-25T14:22:38,090 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53
2024-11-25T14:22:38,193 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:38,209 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=302 (was 302), OpenFileDescriptor=555 (was 555), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=472 (was 472), ProcessCount=11 (was 11), AvailableMemoryMB=4614 (was 4616)
2024-11-25T14:22:38,219 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=302, OpenFileDescriptor=555, MaxFileDescriptor=1048576, SystemLoadAverage=472, ProcessCount=11, AvailableMemoryMB=4614
2024-11-25T14:22:38,220 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54
2024-11-25T14:22:38,222 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?]
	... 12 more
2024-11-25T14:22:38,224 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=54, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:38,225 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=54, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=4 msec
2024-11-25T14:22:38,225 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54
2024-11-25T14:22:38,333 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever
2024-11-25T14:22:38,334 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55
2024-11-25T14:22:38,336 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:38,338 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=55, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:38,339 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=55, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:22:38,339 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55
2024-11-25T14:22:38,443 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:22:38,443 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56
2024-11-25T14:22:38,446 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:38,448 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=56, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:38,449 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=56, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec
2024-11-25T14:22:38,449 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56
2024-11-25T14:22:38,553 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:38,569 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=302 (was 302), OpenFileDescriptor=555 (was 555), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=472 (was 472), ProcessCount=11 (was 11), AvailableMemoryMB=4613 (was 4614)
2024-11-25T14:22:38,578 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=302, OpenFileDescriptor=555, MaxFileDescriptor=1048576, SystemLoadAverage=472, ProcessCount=11, AvailableMemoryMB=4612
2024-11-25T14:22:38,579 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57
2024-11-25T14:22:38,582 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:38,583 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3d3cdc9c to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c219b88
2024-11-25T14:22:38,618 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x3d3cdc9c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3d3cdc9c to 127.0.0.1:51135
2024-11-25T14:22:38,618 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:38,660 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57
2024-11-25T14:22:38,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=58
2024-11-25T14:22:38,813 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:22:38,842 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:38,845 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=58
2024-11-25T14:22:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=58
2024-11-25T14:22:38,848 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:38,851 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57
2024-11-25T14:22:38,851 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec
2024-11-25T14:22:38,851 INFO  [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:22:38,854 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 272 msec
2024-11-25T14:22:38,861 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:38,862 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x13104169 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@78a8d61
2024-11-25T14:22:38,875 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:38,875 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57
2024-11-25T14:22:38,903 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:38,904 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59
2024-11-25T14:22:38,909 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:38,910 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x22302c1d to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@70dec615
2024-11-25T14:22:38,925 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x22302c1d {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x22302c1d to 127.0.0.1:51135
2024-11-25T14:22:38,925 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:38,944 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:39,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59
2024-11-25T14:22:39,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=60
2024-11-25T14:22:39,099 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:39,102 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:39,102 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:39,103 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:39,103 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:39,107 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x48068a30 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@698bd2dc
2024-11-25T14:22:39,107 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:39,123 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:39,123 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:39,124 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60
2024-11-25T14:22:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=60
2024-11-25T14:22:39,127 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:39,130 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59
2024-11-25T14:22:39,130 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec
2024-11-25T14:22:39,130 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:39,133 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 226 msec
2024-11-25T14:22:39,139 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:39,140 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x337369d9 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@10e86c25
2024-11-25T14:22:39,151 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:39,151 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59
2024-11-25T14:22:39,223 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:39,224 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false
2024-11-25T14:22:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61
2024-11-25T14:22:39,228 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:39,229 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x75fba15a to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2438dbf1
2024-11-25T14:22:39,241 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x75fba15a {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x75fba15a to 127.0.0.1:51135
2024-11-25T14:22:39,242 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:39,261 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61
2024-11-25T14:22:39,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=62
2024-11-25T14:22:39,418 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:39,421 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:39,421 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:39,422 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:39,422 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:39,424 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x54115dfe to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1f83b112
2024-11-25T14:22:39,424 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:39,444 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:39,444 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:39,444 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62
2024-11-25T14:22:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=62
2024-11-25T14:22:39,447 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:39,449 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61
2024-11-25T14:22:39,449 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 186 msec
2024-11-25T14:22:39,449 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false
2024-11-25T14:22:39,451 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 225 msec
2024-11-25T14:22:39,460 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:39,462 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x53dc26bd to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2c1f67ac
2024-11-25T14:22:39,467 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:39,468 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61
2024-11-25T14:22:39,542 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:39,543 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:39,544 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-11-25T14:22:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63
2024-11-25T14:22:39,547 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:39,548 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x52eb23ea to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7020e3d2
2024-11-25T14:22:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63
2024-11-25T14:22:39,653 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x52eb23ea {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x52eb23ea to 127.0.0.1:51135
2024-11-25T14:22:39,653 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:39,677 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:39,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=64
2024-11-25T14:22:39,833 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:39,836 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:39,836 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:39,837 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:39,837 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:39,841 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x038f5678 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5987cfba
2024-11-25T14:22:39,841 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:39,862 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63
2024-11-25T14:22:39,862 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:39,862 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=64
2024-11-25T14:22:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=64
2024-11-25T14:22:39,865 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:39,868 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63
2024-11-25T14:22:39,868 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec
2024-11-25T14:22:39,868 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-11-25T14:22:39,871 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 325 msec
2024-11-25T14:22:39,878 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:39,880 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x43919870 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2ecd42c2
2024-11-25T14:22:39,891 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:39,892 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63
2024-11-25T14:22:40,173 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:40,173 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:40,175 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65
2024-11-25T14:22:40,181 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:40,181 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:40,182 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:40,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:40,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:40,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:40,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:40,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:40,193 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65
2024-11-25T14:22:40,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=66
2024-11-25T14:22:40,348 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:22:40,348 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:22:40,349 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:22:40,349 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:40,349 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:40,353 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x02069e1c to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@23c999c0
2024-11-25T14:22:40,354 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:40,354 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:22:40,358 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66
2024-11-25T14:22:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=66
2024-11-25T14:22:40,362 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:40,366 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65
2024-11-25T14:22:40,366 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 169 msec
2024-11-25T14:22:40,366 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:22:40,366 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:22:40,373 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:22:40,375 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 198 msec
2024-11-25T14:22:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65
2024-11-25T14:22:40,494 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:40,495 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67
2024-11-25T14:22:40,501 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:40,502 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=67, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:40,503 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=67, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec
2024-11-25T14:22:40,503 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67
2024-11-25T14:22:40,613 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:22:40,614 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:40,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68
2024-11-25T14:22:40,617 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:40,618 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=68, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:40,620 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=68, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:22:40,620 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68
2024-11-25T14:22:40,723 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:40,738 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=323 (was 302)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x53dc26bd
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x337369d9
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x337369d9-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x43919870-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x48068a30-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x02069e1c-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x54115dfe-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x13104169
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x54115dfe-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x48068a30
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x337369d9-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x54115dfe
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x02069e1c
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x02069e1c-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x13104169-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x53dc26bd-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x038f5678-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x038f5678-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x43919870
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x53dc26bd-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x13104169-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x038f5678
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x43919870-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x48068a30-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
 - Thread LEAK? -, OpenFileDescriptor=583 (was 555) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=472 (was 472), ProcessCount=11 (was 11), AvailableMemoryMB=4584 (was 4612)
2024-11-25T14:22:40,747 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=321, OpenFileDescriptor=583, MaxFileDescriptor=1048576, SystemLoadAverage=472, ProcessCount=11, AvailableMemoryMB=4584
2024-11-25T14:22:40,748 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=69, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69
2024-11-25T14:22:40,752 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:40,753 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x12d5925f to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1fb7557a
2024-11-25T14:22:40,759 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x12d5925f {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x12d5925f to 127.0.0.1:51135
2024-11-25T14:22:40,759 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:40,786 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69
2024-11-25T14:22:40,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=70
2024-11-25T14:22:40,941 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:22:40,981 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:40,984 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=70
2024-11-25T14:22:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=70
2024-11-25T14:22:40,987 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:40,989 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69
2024-11-25T14:22:40,989 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 201 msec
2024-11-25T14:22:40,989 INFO  [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:22:40,991 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 241 msec
2024-11-25T14:22:41,000 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:41,001 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1c9ea373 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@153a6725
2024-11-25T14:22:41,009 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:41,009 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69
2024-11-25T14:22:41,072 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:41,073 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=71, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:41,077 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71
2024-11-25T14:22:41,078 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0491fedd to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1d7f029b
2024-11-25T14:22:41,157 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x0491fedd {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0491fedd to 127.0.0.1:51135
2024-11-25T14:22:41,157 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71
2024-11-25T14:22:41,185 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:41,203 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-25T14:22:41,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=72
2024-11-25T14:22:41,340 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:41,343 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:41,343 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:41,343 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:41,344 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:41,347 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x455f01b0 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3e361d86
2024-11-25T14:22:41,347 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:41,368 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:41,368 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:41,368 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=72
2024-11-25T14:22:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=72
2024-11-25T14:22:41,371 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:41,373 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=71
2024-11-25T14:22:41,374 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:41,374 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 186 msec
2024-11-25T14:22:41,376 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 301 msec
2024-11-25T14:22:41,384 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:41,386 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1ecc2f60 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@9afc212
2024-11-25T14:22:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71
2024-11-25T14:22:41,392 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:41,392 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:41,393 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:41,393 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:41,394 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false
2024-11-25T14:22:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=73, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73
2024-11-25T14:22:41,397 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:41,399 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7aae3999 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e642113
2024-11-25T14:22:41,409 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x7aae3999 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7aae3999 to 127.0.0.1:51135
2024-11-25T14:22:41,409 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:41,427 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73
2024-11-25T14:22:41,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=74
2024-11-25T14:22:41,583 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:41,586 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:41,587 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:41,587 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:41,588 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:41,592 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x518fa6e6 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@114fb8f6
2024-11-25T14:22:41,593 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:41,608 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:41,608 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:41,609 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=74
2024-11-25T14:22:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=74
2024-11-25T14:22:41,612 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:41,614 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73
2024-11-25T14:22:41,614 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false
2024-11-25T14:22:41,614 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 185 msec
2024-11-25T14:22:41,616 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 220 msec
2024-11-25T14:22:41,624 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:41,625 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5ec2938d to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d4cd159
2024-11-25T14:22:41,634 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:41,634 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:41,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73
2024-11-25T14:22:41,712 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:41,713 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:41,714 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:41,715 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-11-25T14:22:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=75, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75
2024-11-25T14:22:41,718 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:41,720 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x46badc7d to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4a357456
2024-11-25T14:22:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22
2024-11-25T14:22:41,726 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x46badc7d {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x46badc7d to 127.0.0.1:51135
2024-11-25T14:22:41,726 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:41,744 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75
2024-11-25T14:22:41,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=76
2024-11-25T14:22:41,898 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:41,901 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:41,901 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:41,901 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:41,901 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:41,904 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x05c4b16a to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5e073574
2024-11-25T14:22:41,905 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:41,923 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:41,924 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:41,924 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=76
2024-11-25T14:22:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=76
2024-11-25T14:22:41,928 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:41,930 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75
2024-11-25T14:22:41,931 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-11-25T14:22:41,931 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 184 msec
2024-11-25T14:22:41,933 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 216 msec
2024-11-25T14:22:41,942 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:41,943 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2264f4e0 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@584a81d5
2024-11-25T14:22:41,951 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:41,951 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75
2024-11-25T14:22:42,033 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:42,033 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:42,036 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:42,037 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-11-25T14:22:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=77, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77
2024-11-25T14:22:42,041 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:42,042 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x08f10a59 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2a9059fe
2024-11-25T14:22:42,051 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x08f10a59 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x08f10a59 to 127.0.0.1:51135
2024-11-25T14:22:42,051 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:42,068 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77
2024-11-25T14:22:42,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=78
2024-11-25T14:22:42,222 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:42,223 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:42,223 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:42,223 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:42,224 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:42,225 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2fb64646 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@553adccc
2024-11-25T14:22:42,225 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:42,243 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:42,243 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:42,244 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=78
2024-11-25T14:22:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=78
2024-11-25T14:22:42,247 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:42,249 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77
2024-11-25T14:22:42,249 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec
2024-11-25T14:22:42,249 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-11-25T14:22:42,251 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 211 msec
2024-11-25T14:22:42,264 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:42,265 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x078553ef to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@377c814c
2024-11-25T14:22:42,276 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:42,276 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77
2024-11-25T14:22:42,363 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:42,363 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:42,365 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:42,367 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-25T14:22:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79
2024-11-25T14:22:42,370 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:42,371 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a9f3ef1 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@27e8cdf6
2024-11-25T14:22:42,385 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x7a9f3ef1 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a9f3ef1 to 127.0.0.1:51135
2024-11-25T14:22:42,385 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:42,403 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79
2024-11-25T14:22:42,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=80
2024-11-25T14:22:42,557 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:42,558 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:42,559 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:42,559 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:42,559 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:42,561 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7d556c4b to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d582c29
2024-11-25T14:22:42,561 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:42,580 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:42,580 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:42,580 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=80
2024-11-25T14:22:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=80
2024-11-25T14:22:42,584 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:42,587 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79
2024-11-25T14:22:42,587 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 182 msec
2024-11-25T14:22:42,587 INFO  [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-25T14:22:42,589 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 220 msec
2024-11-25T14:22:42,597 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:42,598 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x41cde2ca to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7be063ab
2024-11-25T14:22:42,609 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:42,609 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79
2024-11-25T14:22:42,693 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:42,694 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:42,696 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:42,697 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-25T14:22:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81
2024-11-25T14:22:42,700 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:42,701 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69f7846e to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3987d204
2024-11-25T14:22:42,709 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x69f7846e {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x69f7846e to 127.0.0.1:51135
2024-11-25T14:22:42,709 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:42,726 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81
2024-11-25T14:22:42,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=82
2024-11-25T14:22:42,881 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:42,884 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:42,884 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:42,884 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:42,885 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:42,887 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3e6b11fa to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4e608f4a
2024-11-25T14:22:42,888 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:42,905 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:42,906 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:42,906 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82
2024-11-25T14:22:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=82
2024-11-25T14:22:42,909 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:42,911 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81
2024-11-25T14:22:42,911 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec
2024-11-25T14:22:42,911 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-25T14:22:42,913 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 214 msec
2024-11-25T14:22:42,922 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:42,923 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d31d9c3 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@54a41497
2024-11-25T14:22:42,934 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:42,934 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81
2024-11-25T14:22:43,024 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:43,024 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:43,026 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-25T14:22:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83
2024-11-25T14:22:43,030 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:43,031 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x13a1a3cc to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@294a3bba
2024-11-25T14:22:43,042 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x13a1a3cc {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x13a1a3cc to 127.0.0.1:51135
2024-11-25T14:22:43,042 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:43,060 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83
2024-11-25T14:22:43,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=84
2024-11-25T14:22:43,214 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:43,215 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84
2024-11-25T14:22:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=84
2024-11-25T14:22:43,218 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:43,220 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83
2024-11-25T14:22:43,220 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 158 msec
2024-11-25T14:22:43,220 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-25T14:22:43,222 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 194 msec
2024-11-25T14:22:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83
2024-11-25T14:22:43,342 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:43,343 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:43,345 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:43,346 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false
2024-11-25T14:22:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:43,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85
2024-11-25T14:22:43,350 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:43,351 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ba0b163 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4656af7
2024-11-25T14:22:43,359 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x7ba0b163 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7ba0b163 to 127.0.0.1:51135
2024-11-25T14:22:43,359 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:43,376 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85
2024-11-25T14:22:43,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=86
2024-11-25T14:22:43,531 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:43,535 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:43,535 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:43,536 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:43,536 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:43,539 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x01767458 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1ca52d8b
2024-11-25T14:22:43,539 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:43,559 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:43,559 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:43,560 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86
2024-11-25T14:22:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=86
2024-11-25T14:22:43,562 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:43,564 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85
2024-11-25T14:22:43,564 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false
2024-11-25T14:22:43,564 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 186 msec
2024-11-25T14:22:43,566 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 218 msec
2024-11-25T14:22:43,576 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:43,578 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3a88f268 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3d91e0d0
2024-11-25T14:22:43,601 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:43,601 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85
2024-11-25T14:22:43,663 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:43,664 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:43,667 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false
2024-11-25T14:22:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87
2024-11-25T14:22:43,671 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:43,672 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0abc8e99 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@52ab5d03
2024-11-25T14:22:43,684 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x0abc8e99 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0abc8e99 to 127.0.0.1:51135
2024-11-25T14:22:43,685 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:43,701 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87
2024-11-25T14:22:43,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=88
2024-11-25T14:22:43,854 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:43,855 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:43,855 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:43,855 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:43,856 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:43,857 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x42c654df to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e3a584b
2024-11-25T14:22:43,857 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:43,877 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:43,877 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:43,878 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88
2024-11-25T14:22:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=88
2024-11-25T14:22:43,880 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:43,883 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87
2024-11-25T14:22:43,883 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec
2024-11-25T14:22:43,883 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false
2024-11-25T14:22:43,885 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 216 msec
2024-11-25T14:22:43,894 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:43,896 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4d7bc86b to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7687f184
2024-11-25T14:22:43,908 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:43,908 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87
2024-11-25T14:22:43,992 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:43,993 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:43,994 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89
2024-11-25T14:22:43,999 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:44,000 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:44,000 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:44,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:44,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:44,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:44,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:44,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:44,010 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89
2024-11-25T14:22:44,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=90
2024-11-25T14:22:44,163 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:22:44,163 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:22:44,163 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:22:44,164 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:44,164 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:44,165 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x47c649f4 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3b5a31ae
2024-11-25T14:22:44,166 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:44,166 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:22:44,169 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90
2024-11-25T14:22:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=90
2024-11-25T14:22:44,171 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:44,173 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89
2024-11-25T14:22:44,173 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:22:44,174 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:22:44,174 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 161 msec
2024-11-25T14:22:44,180 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:22:44,182 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 185 msec
2024-11-25T14:22:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89
2024-11-25T14:22:44,313 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:44,313 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91
2024-11-25T14:22:44,316 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:44,317 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=91, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:44,318 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=91, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:22:44,318 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91
2024-11-25T14:22:44,423 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:22:44,424 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92
2024-11-25T14:22:44,428 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:44,429 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=92, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:44,431 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=92, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec
2024-11-25T14:22:44,431 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92
2024-11-25T14:22:44,532 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:44,550 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=376 (was 321)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x01767458-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x5ec2938d
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x47c649f4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x41cde2ca-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x01767458-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x3e6b11fa-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7d556c4b-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x518fa6e6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x5ec2938d-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x3e6b11fa-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x47c649f4-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4d7bc86b-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x5ec2938d-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1ecc2f60-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x078553ef
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x518fa6e6-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x41cde2ca
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1c9ea373
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x3a88f268-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x47c649f4-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x455f01b0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0d31d9c3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x41cde2ca-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x42c654df-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x3e6b11fa
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2264f4e0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0d31d9c3-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2fb64646-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x455f01b0-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x05c4b16a-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x01767458
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x42c654df
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x455f01b0-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2fb64646
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1ecc2f60-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x42c654df-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1c9ea373-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2fb64646-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x05c4b16a
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1c9ea373-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1ecc2f60
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4d7bc86b
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2264f4e0-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x3a88f268-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x078553ef-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x518fa6e6-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0d31d9c3-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x078553ef-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4d7bc86b-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x05c4b16a-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x3a88f268
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2264f4e0-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7d556c4b
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7d556c4b-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
 - Thread LEAK? -, OpenFileDescriptor=653 (was 583) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=443 (was 472), ProcessCount=11 (was 11), AvailableMemoryMB=4452 (was 4584)
2024-11-25T14:22:44,565 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=376, OpenFileDescriptor=653, MaxFileDescriptor=1048576, SystemLoadAverage=443, ProcessCount=11, AvailableMemoryMB=4450
2024-11-25T14:22:44,567 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93
2024-11-25T14:22:44,571 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:44,572 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x76cf99cf to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@58bec02c
2024-11-25T14:22:44,584 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x76cf99cf {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x76cf99cf to 127.0.0.1:51135
2024-11-25T14:22:44,584 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93
2024-11-25T14:22:44,744 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93
2024-11-25T14:22:44,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=94
2024-11-25T14:22:44,898 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:22:44,928 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:44,931 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94
2024-11-25T14:22:44,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=94
2024-11-25T14:22:44,935 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:44,937 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93
2024-11-25T14:22:44,937 INFO  [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:22:44,937 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec
2024-11-25T14:22:44,939 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 370 msec
2024-11-25T14:22:44,949 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:44,951 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5f6ea0e3 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7b509820
2024-11-25T14:22:45,044 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:45,044 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:45,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93
2024-11-25T14:22:45,202 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:22:45,207 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3992): Client=jenkins//172.17.0.3 disable replication peer, id=1
2024-11-25T14:22:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure
2024-11-25T14:22:45,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95
2024-11-25T14:22:45,258 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:45,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95
2024-11-25T14:22:45,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=96
2024-11-25T14:22:45,416 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=DISABLE_PEER
2024-11-25T14:22:45,417 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:45,417 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:45,417 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:45,417 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:45,418 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x48efa64e to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7353b9d0
2024-11-25T14:22:45,419 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:45,440 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:45,440 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:45,441 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96
2024-11-25T14:22:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=96
2024-11-25T14:22:45,443 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for DISABLE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:45,446 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95
2024-11-25T14:22:45,446 INFO  [PEWorker-4 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1
2024-11-25T14:22:45,446 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 185 msec
2024-11-25T14:22:45,448 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 239 msec
2024-11-25T14:22:45,458 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:45,459 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x10322b42 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4feafac7
2024-11-25T14:22:45,518 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:45,519 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95
2024-11-25T14:22:45,532 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:22:45,534 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97
2024-11-25T14:22:45,537 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:45,538 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:45,538 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:45,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:45,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:45,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:45,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:45,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:45,560 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97
2024-11-25T14:22:45,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=98
2024-11-25T14:22:45,713 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:22:45,714 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:22:45,714 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:22:45,714 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:45,714 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:45,716 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x70f0cfdb to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@74c2b9c8
2024-11-25T14:22:45,717 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:45,717 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:22:45,721 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98
2024-11-25T14:22:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=98
2024-11-25T14:22:45,724 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:45,726 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97
2024-11-25T14:22:45,727 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:22:45,727 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 164 msec
2024-11-25T14:22:45,727 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:22:45,735 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:22:45,737 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 201 msec
2024-11-25T14:22:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97
2024-11-25T14:22:45,853 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:45,853 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99
2024-11-25T14:22:45,856 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:45,857 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=99, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:45,858 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=99, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:22:45,858 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99
2024-11-25T14:22:45,963 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:22:45,964 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100
2024-11-25T14:22:45,969 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:45,972 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=100, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:45,974 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=100, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=8 msec
2024-11-25T14:22:45,974 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100
2024-11-25T14:22:46,084 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:46,104 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=388 (was 376)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x10322b42-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x10322b42-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x48efa64e
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x10322b42
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x5f6ea0e3-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x5f6ea0e3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x70f0cfdb
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x5f6ea0e3-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x48efa64e-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x48efa64e-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x70f0cfdb-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x70f0cfdb-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
 - Thread LEAK? -, OpenFileDescriptor=669 (was 653) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=443 (was 443), ProcessCount=11 (was 11), AvailableMemoryMB=5185 (was 4450) - AvailableMemoryMB LEAK? -
2024-11-25T14:22:46,115 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=388, OpenFileDescriptor=669, MaxFileDescriptor=1048576, SystemLoadAverage=443, ProcessCount=11, AvailableMemoryMB=5182
2024-11-25T14:22:46,116 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=dummy_peer
2024-11-25T14:22:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101
2024-11-25T14:22:46,118 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:46,119 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=101, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:46,120 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=101, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-25T14:22:46,120 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101
2024-11-25T14:22:46,222 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer
2024-11-25T14:22:46,223 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=102, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102
2024-11-25T14:22:46,226 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:46,227 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=102, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:46,229 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=102, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec
2024-11-25T14:22:46,229 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102
2024-11-25T14:22:46,333 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:22:46,334 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103
2024-11-25T14:22:46,339 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:46,341 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=103, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:46,343 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=103, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec
2024-11-25T14:22:46,343 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103
2024-11-25T14:22:46,443 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:46,463 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=388 (was 388), OpenFileDescriptor=669 (was 669), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=443 (was 443), ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=5124 (was 5182)
2024-11-25T14:22:46,474 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=388, OpenFileDescriptor=669, MaxFileDescriptor=1048576, SystemLoadAverage=443, ProcessCount=11, AvailableMemoryMB=5123
2024-11-25T14:22:46,476 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104
2024-11-25T14:22:46,480 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:46,482 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6c7a61e7 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@59322625
2024-11-25T14:22:46,501 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x6c7a61e7 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6c7a61e7 to 127.0.0.1:51135
2024-11-25T14:22:46,501 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:46,526 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104
2024-11-25T14:22:46,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=105
2024-11-25T14:22:46,680 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:22:46,722 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:46,726 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=105
2024-11-25T14:22:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=105
2024-11-25T14:22:46,729 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:46,731 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104
2024-11-25T14:22:46,732 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 203 msec
2024-11-25T14:22:46,731 INFO  [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:22:46,734 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 256 msec
2024-11-25T14:22:46,749 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:46,751 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x46a001ec to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@730541a1
2024-11-25T14:22:46,786 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:46,786 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104
2024-11-25T14:22:46,792 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:22:46,794 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106
2024-11-25T14:22:46,799 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:46,799 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:46,800 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:46,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:46,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:46,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:46,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:46,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:46,818 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106
2024-11-25T14:22:46,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=107
2024-11-25T14:22:46,973 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:22:46,973 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:22:46,974 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:22:46,974 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:46,974 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:46,976 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2cad830f to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@39611f10
2024-11-25T14:22:46,977 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:46,978 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:22:46,984 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=107
2024-11-25T14:22:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=107
2024-11-25T14:22:46,987 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:46,990 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106
2024-11-25T14:22:46,990 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:22:46,990 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:22:46,990 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 169 msec
2024-11-25T14:22:46,998 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:22:47,005 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 204 msec
2024-11-25T14:22:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106
2024-11-25T14:22:47,112 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:47,113 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108
2024-11-25T14:22:47,116 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:47,117 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=108, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:47,118 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=108, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:22:47,118 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108
2024-11-25T14:22:47,223 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:22:47,225 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109
2024-11-25T14:22:47,231 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:47,234 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=109, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:47,236 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=109, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=9 msec
2024-11-25T14:22:47,236 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:47,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109
2024-11-25T14:22:47,333 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:47,361 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=394 (was 388)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2cad830f-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2cad830f
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2cad830f-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x46a001ec
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x46a001ec-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x46a001ec-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
 - Thread LEAK? -, OpenFileDescriptor=677 (was 669) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=443 (was 443), ProcessCount=11 (was 11), AvailableMemoryMB=5088 (was 5123)
2024-11-25T14:22:47,385 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=394, OpenFileDescriptor=677, MaxFileDescriptor=1048576, SystemLoadAverage=443, ProcessCount=11, AvailableMemoryMB=5084
2024-11-25T14:22:47,395 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] ipc.MetricsHBaseServer(152): Unknown exception type
java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] ipc.CallRunner(138): callId: 314 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.3:37128 deadline: 1732544627394, exception=java.io.IOException: Replication peer modification disabled
2024-11-25T14:22:47,421 WARN  [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 25 ms
java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?]
	at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 32 more
2024-11-25T14:22:47,532 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] ipc.MetricsHBaseServer(152): Unknown exception type
java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] ipc.CallRunner(138): callId: 315 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.3:37128 deadline: 1732544627532, exception=java.io.IOException: Replication peer modification disabled
2024-11-25T14:22:47,533 WARN  [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 139 ms
java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?]
	at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 32 more
2024-11-25T14:22:47,535 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions:
2024-11-25T14:22:47.423Z, java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-25T14:22:47.534Z, java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)


2024-11-25T14:22:47,538 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110
2024-11-25T14:22:47,541 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:47,542 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=110, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:47,543 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=110, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:22:47,543 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110
2024-11-25T14:22:47,652 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:22:47,653 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111
2024-11-25T14:22:47,655 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:47,656 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=111, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:47,658 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=111, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:22:47,658 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111
2024-11-25T14:22:47,762 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:47,779 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=394 (was 394), OpenFileDescriptor=677 (was 677), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 443) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5160 (was 5084) - AvailableMemoryMB LEAK? -
2024-11-25T14:22:47,790 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=394, OpenFileDescriptor=677, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=5142
2024-11-25T14:22:47,792 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112
2024-11-25T14:22:47,795 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:47,796 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58510021 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@37b70c6b
2024-11-25T14:22:47,857 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x58510021 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x58510021 to 127.0.0.1:51135
2024-11-25T14:22:47,857 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112
2024-11-25T14:22:48,112 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112
2024-11-25T14:22:48,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=113
2024-11-25T14:22:48,265 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:22:48,313 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:48,320 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=113
2024-11-25T14:22:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=113
2024-11-25T14:22:48,326 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:48,331 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112
2024-11-25T14:22:48,331 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 215 msec
2024-11-25T14:22:48,331 INFO  [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:22:48,339 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 539 msec
2024-11-25T14:22:48,347 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:48,350 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2367d593 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@103153dc
2024-11-25T14:22:48,388 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:48,389 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112
2024-11-25T14:22:48,422 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:48,423 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:48,424 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114
2024-11-25T14:22:48,428 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:48,428 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:48,428 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:48,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:48,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:48,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:48,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:48,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:48,443 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114
2024-11-25T14:22:48,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=115
2024-11-25T14:22:48,596 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:22:48,596 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:22:48,596 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:22:48,596 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:48,596 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:48,599 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4ddd7d65 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1569c9f7
2024-11-25T14:22:48,599 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:48,599 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:22:48,602 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=115
2024-11-25T14:22:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=115
2024-11-25T14:22:48,610 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:48,612 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114
2024-11-25T14:22:48,612 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:22:48,612 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 167 msec
2024-11-25T14:22:48,612 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:22:48,620 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:22:48,621 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 196 msec
2024-11-25T14:22:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114
2024-11-25T14:22:48,742 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:48,743 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116
2024-11-25T14:22:48,754 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:48,756 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=116, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:48,758 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=116, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=13 msec
2024-11-25T14:22:48,758 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116
2024-11-25T14:22:48,863 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:48,882 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=400 (was 394)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2367d593-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4ddd7d65
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2367d593
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4ddd7d65-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4ddd7d65-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2367d593-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
 - Thread LEAK? -, OpenFileDescriptor=685 (was 677) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=5097 (was 5142)
2024-11-25T14:22:48,896 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=400, OpenFileDescriptor=685, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=5094
2024-11-25T14:22:48,897 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:48,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117
2024-11-25T14:22:48,901 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:48,902 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x16a08e67 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@51978335
2024-11-25T14:22:48,917 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x16a08e67 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x16a08e67 to 127.0.0.1:51135
2024-11-25T14:22:48,917 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:48,943 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:49,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117
2024-11-25T14:22:49,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=118
2024-11-25T14:22:49,097 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:22:49,134 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:49,137 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118
2024-11-25T14:22:49,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=118
2024-11-25T14:22:49,140 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:49,144 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117
2024-11-25T14:22:49,144 INFO  [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:22:49,144 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 198 msec
2024-11-25T14:22:49,147 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 247 msec
2024-11-25T14:22:49,156 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:49,157 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x53d9b9af to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1de8615e
2024-11-25T14:22:49,170 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:49,170 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:49,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117
2024-11-25T14:22:49,224 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:49,225 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:49,227 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false
2024-11-25T14:22:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:49,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119
2024-11-25T14:22:49,240 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:49,242 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5dafbf80 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5f038fe9
2024-11-25T14:22:49,264 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x5dafbf80 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5dafbf80 to 127.0.0.1:51135
2024-11-25T14:22:49,267 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:49,285 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:49,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119
2024-11-25T14:22:49,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=120
2024-11-25T14:22:49,439 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:49,441 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120
2024-11-25T14:22:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=120
2024-11-25T14:22:49,443 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:49,446 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119
2024-11-25T14:22:49,446 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false
2024-11-25T14:22:49,447 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 158 msec
2024-11-25T14:22:49,449 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 220 msec
2024-11-25T14:22:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119
2024-11-25T14:22:49,553 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:49,554 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:49,555 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121
2024-11-25T14:22:49,561 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:49,561 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:49,562 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:49,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:49,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:49,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:49,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:49,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:49,591 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121
2024-11-25T14:22:49,669 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-25T14:22:49,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=122
2024-11-25T14:22:49,755 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:22:49,755 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:22:49,755 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:22:49,755 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:49,756 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:49,758 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7fdcec6d to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@754dd07e
2024-11-25T14:22:49,759 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:49,760 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:22:49,768 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122
2024-11-25T14:22:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=122
2024-11-25T14:22:49,772 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:49,778 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121
2024-11-25T14:22:49,778 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:22:49,778 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 181 msec
2024-11-25T14:22:49,778 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:22:49,788 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:22:49,790 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 233 msec
2024-11-25T14:22:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121
2024-11-25T14:22:49,872 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:49,873 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123
2024-11-25T14:22:49,875 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:49,876 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=123, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:49,877 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=123, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-25T14:22:49,877 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123
2024-11-25T14:22:49,982 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:22:49,983 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:49,986 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124
2024-11-25T14:22:49,987 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=124, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:49,988 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=124, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:22:49,988 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124
2024-11-25T14:22:50,093 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:50,121 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=406 (was 400)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x53d9b9af
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7fdcec6d-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7fdcec6d-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x53d9b9af-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7fdcec6d
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x53d9b9af-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
 - Thread LEAK? -, OpenFileDescriptor=693 (was 685) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=4951 (was 5094)
2024-11-25T14:22:50,148 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=406, OpenFileDescriptor=693, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=5176
2024-11-25T14:22:50,152 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125
2024-11-25T14:22:50,239 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:50,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125
2024-11-25T14:22:50,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=126
2024-11-25T14:22:50,398 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:22:50,471 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:50,480 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126
2024-11-25T14:22:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=126
2024-11-25T14:22:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125
2024-11-25T14:22:50,483 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(613): peerId=1, queueId=1-cb69081f5976,36701,1732544544686 (queues=1) is replicating from cluster=b9c5d22a-5299-4650-a1d7-40bf093ed607 to cluster=b9c5d22a-5299-4650-a1d7-40bf093ed607
2024-11-25T14:22:50,485 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(382): peerId=1, starting shipping worker for walGroupId=cb69081f5976%2C36701%2C1732544544686
2024-11-25T14:22:50,496 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=1-cb69081f5976,36701,1732544544686, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1
2024-11-25T14:22:50,496 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:50,503 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125
2024-11-25T14:22:50,503 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 257 msec
2024-11-25T14:22:50,503 INFO  [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:22:50,506 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 351 msec
2024-11-25T14:22:50,507 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686.replicationSource.wal-reader.cb69081f5976%2C36701%2C1732544544686,1-cb69081f5976,36701,1732544544686 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/WALs/cb69081f5976,36701,1732544544686/cb69081f5976%2C36701%2C1732544544686.1732544547907, startPosition=0, beingWritten=true
2024-11-25T14:22:50,507 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686.replicationSource.shippercb69081f5976%2C36701%2C1732544544686,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: cb69081f5976%2C36701%2C1732544544686
2024-11-25T14:22:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125
2024-11-25T14:22:50,796 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:50,800 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127
2024-11-25T14:22:50,807 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: 
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: java.io.IOException: Cluster key passed  is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?]
	... 12 more
2024-11-25T14:22:50,810 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=127, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: 
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: 
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
Caused by: java.io.IOException: Cluster key passed  is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:50,812 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=127, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=10 msec
2024-11-25T14:22:50,813 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127
2024-11-25T14:22:50,913 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 
2024-11-25T14:22:50,916 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128
2024-11-25T14:22:50,943 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:50,949 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:50,955 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:51,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:51,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:51,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:51,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:51,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:51,005 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128
2024-11-25T14:22:51,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=129
2024-11-25T14:22:51,169 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:22:51,169 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:22:51,169 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:22:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128
2024-11-25T14:22:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128
2024-11-25T14:22:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22
2024-11-25T14:22:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128
2024-11-25T14:22:52,171 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686.replicationSource.wal-reader.cb69081f5976%2C36701%2C1732544544686,1-cb69081f5976,36701,1732544544686 {}] util.Threads(127): sleep interrupted
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) ~[classes/:?]
2024-11-25T14:22:52,172 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686.replicationSource.shippercb69081f5976%2C36701%2C1732544544686,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch
java.lang.InterruptedException: null
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?]
	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:309) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?]
2024-11-25T14:22:52,894 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-25T14:22:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128
2024-11-25T14:22:53,175 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSource(739): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686.replicationSource.shippercb69081f5976%2C36701%2C1732544544686,1-cb69081f5976,36701,1732544544686 terminated
2024-11-25T14:22:53,175 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:22:53,181 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=129
2024-11-25T14:22:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=129
2024-11-25T14:22:53,192 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:53,199 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128
2024-11-25T14:22:53,199 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:22:53,199 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:22:53,200 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1880 sec
2024-11-25T14:22:53,232 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:22:53,242 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.3160 sec
2024-11-25T14:22:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128
2024-11-25T14:22:55,093 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:55,094 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:55,096 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130
2024-11-25T14:22:55,100 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=130, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:55,101 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=130, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec
2024-11-25T14:22:55,102 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130
2024-11-25T14:22:55,213 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:55,234 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=409 (was 406) - Thread LEAK? -, OpenFileDescriptor=700 (was 693) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=472 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=5331 (was 5176) - AvailableMemoryMB LEAK? -
2024-11-25T14:22:55,245 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=409, OpenFileDescriptor=700, MaxFileDescriptor=1048576, SystemLoadAverage=472, ProcessCount=11, AvailableMemoryMB=5329
2024-11-25T14:22:55,247 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131
2024-11-25T14:22:55,250 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:55,251 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5e9de1da to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@31a1df56
2024-11-25T14:22:55,262 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-25T14:22:55,308 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x5e9de1da {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5e9de1da to 127.0.0.1:51135
2024-11-25T14:22:55,309 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:55,344 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131
2024-11-25T14:22:55,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=132
2024-11-25T14:22:55,503 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:22:55,559 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131
2024-11-25T14:22:55,563 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132
2024-11-25T14:22:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=132
2024-11-25T14:22:55,576 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:55,585 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131
2024-11-25T14:22:55,586 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 232 msec
2024-11-25T14:22:55,586 INFO  [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:22:55,587 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:55,588 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 339 msec
2024-11-25T14:22:55,596 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x17fd04ec to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@15654964
2024-11-25T14:22:55,617 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:55,617 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131
2024-11-25T14:22:55,872 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:55,873 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133
2024-11-25T14:22:55,877 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:55,878 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x48f95b09 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@65abe773
2024-11-25T14:22:55,899 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x48f95b09 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x48f95b09 to 127.0.0.1:51135
2024-11-25T14:22:55,899 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:55,917 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133
2024-11-25T14:22:56,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=134
2024-11-25T14:22:56,070 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:56,071 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:56,071 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:56,072 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:56,072 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:56,073 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4eb56ec3 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5eaf045f
2024-11-25T14:22:56,074 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:56,100 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:56,101 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:56,103 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134
2024-11-25T14:22:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=134
2024-11-25T14:22:56,108 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:56,111 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133
2024-11-25T14:22:56,111 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec
2024-11-25T14:22:56,111 INFO  [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:56,114 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 238 msec
2024-11-25T14:22:56,147 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:56,157 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7dbbec35 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@15d081df
2024-11-25T14:22:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133
2024-11-25T14:22:56,196 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:56,195 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:56,196 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:56,211 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:56,218 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:56,219 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:56,221 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false
2024-11-25T14:22:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:56,225 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:56,229 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x297994de to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@c27c2f
2024-11-25T14:22:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135
2024-11-25T14:22:56,291 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x297994de {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x297994de to 127.0.0.1:51135
2024-11-25T14:22:56,292 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:56,302 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135
2024-11-25T14:22:56,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=136
2024-11-25T14:22:56,456 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:56,458 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:56,458 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:56,458 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:56,458 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:56,460 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d3e603a to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7cce8edb
2024-11-25T14:22:56,460 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:56,476 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:56,477 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:56,477 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136
2024-11-25T14:22:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=136
2024-11-25T14:22:56,479 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:56,481 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135
2024-11-25T14:22:56,481 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false
2024-11-25T14:22:56,481 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec
2024-11-25T14:22:56,482 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 260 msec
2024-11-25T14:22:56,497 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:56,499 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5fc5b243 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4fe91d3
2024-11-25T14:22:56,508 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:56,509 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135
2024-11-25T14:22:56,553 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:56,553 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:56,555 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:56,556 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:56,557 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:56,558 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false
2024-11-25T14:22:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137
2024-11-25T14:22:56,561 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:56,562 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a1e5543 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72a90e6
2024-11-25T14:22:56,574 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x7a1e5543 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a1e5543 to 127.0.0.1:51135
2024-11-25T14:22:56,574 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:56,593 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137
2024-11-25T14:22:56,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=138
2024-11-25T14:22:56,746 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:56,748 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:56,748 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:56,748 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:56,748 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:56,750 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x19255325 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@a14278
2024-11-25T14:22:56,750 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:56,771 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:56,771 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:56,771 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138
2024-11-25T14:22:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=138
2024-11-25T14:22:56,774 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:56,776 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137
2024-11-25T14:22:56,776 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false
2024-11-25T14:22:56,776 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 182 msec
2024-11-25T14:22:56,777 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 218 msec
2024-11-25T14:22:56,787 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:56,788 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x67e378ba to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2a655101
2024-11-25T14:22:56,800 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:56,800 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137
2024-11-25T14:22:56,882 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:56,882 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:56,883 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:56,884 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:56,885 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139
2024-11-25T14:22:56,888 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:56,889 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6934006d to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2841e049
2024-11-25T14:22:56,900 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x6934006d {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6934006d to 127.0.0.1:51135
2024-11-25T14:22:56,900 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:56,918 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139
2024-11-25T14:22:57,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=140
2024-11-25T14:22:57,072 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:57,073 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:57,073 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:57,073 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:57,073 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:57,074 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2a17d2ee to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1bde2469
2024-11-25T14:22:57,075 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:57,092 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:57,092 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:57,093 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140
2024-11-25T14:22:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=140
2024-11-25T14:22:57,095 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:57,097 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139
2024-11-25T14:22:57,097 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:57,097 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 178 msec
2024-11-25T14:22:57,098 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 212 msec
2024-11-25T14:22:57,112 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:57,114 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x42d4b10c to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@64361b11
2024-11-25T14:22:57,125 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:57,125 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139
2024-11-25T14:22:57,212 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:57,213 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:57,213 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:57,214 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false
2024-11-25T14:22:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141
2024-11-25T14:22:57,217 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:57,218 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a8a3cee to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4e37726f
2024-11-25T14:22:57,225 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x7a8a3cee {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a8a3cee to 127.0.0.1:51135
2024-11-25T14:22:57,225 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:57,242 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141
2024-11-25T14:22:57,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=142
2024-11-25T14:22:57,396 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:57,397 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:57,397 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:57,397 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:57,397 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:57,399 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x156d8469 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7c4f75ec
2024-11-25T14:22:57,399 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:57,421 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:57,422 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:57,422 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142
2024-11-25T14:22:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=142
2024-11-25T14:22:57,427 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:57,429 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141
2024-11-25T14:22:57,429 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false
2024-11-25T14:22:57,429 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 185 msec
2024-11-25T14:22:57,431 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 215 msec
2024-11-25T14:22:57,443 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:57,445 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x110aae1f to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@117f55b4
2024-11-25T14:22:57,450 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:57,450 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141
2024-11-25T14:22:57,533 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:57,534 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:57,535 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143
2024-11-25T14:22:57,541 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:57,542 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0526e0c0 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@216db783
2024-11-25T14:22:57,551 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x0526e0c0 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0526e0c0 to 127.0.0.1:51135
2024-11-25T14:22:57,552 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:57,567 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143
2024-11-25T14:22:57,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=144
2024-11-25T14:22:57,722 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:57,723 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:57,723 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:57,723 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:57,723 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:57,724 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x461585ca to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e60cb93
2024-11-25T14:22:57,725 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:57,746 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:57,747 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:57,747 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144
2024-11-25T14:22:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=144
2024-11-25T14:22:57,750 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:57,752 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143
2024-11-25T14:22:57,752 INFO  [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:57,752 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec
2024-11-25T14:22:57,757 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 217 msec
2024-11-25T14:22:57,763 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:57,764 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b949a6a to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7f2fcce9
2024-11-25T14:22:57,775 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:57,775 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143
2024-11-25T14:22:57,872 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:57,873 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:22:57,874 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:57,874 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145
2024-11-25T14:22:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-11-25T14:22:57,878 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:57,879 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:57,879 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:57,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:57,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:57,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:57,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:57,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:57,892 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:57,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145
2024-11-25T14:22:57,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-11-25T14:22:58,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=147
2024-11-25T14:22:58,045 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:22:58,046 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:22:58,046 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:22:58,046 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:58,046 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:58,048 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x35a043cb to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@42c50087
2024-11-25T14:22:58,049 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:58,049 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:22:58,051 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147
2024-11-25T14:22:58,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=147
2024-11-25T14:22:58,053 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:58,055 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145
2024-11-25T14:22:58,055 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 161 msec
2024-11-25T14:22:58,055 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:22:58,055 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:22:58,063 INFO  [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:22:58,064 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 189 msec
2024-11-25T14:22:58,065 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:58,066 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=146, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:58,067 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=146, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=192 msec
2024-11-25T14:22:58,067 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145
2024-11-25T14:22:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146
2024-11-25T14:22:58,192 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:58,193 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:22:58,193 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:22:58,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:58,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148
2024-11-25T14:22:58,196 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:58,197 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=148, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:58,198 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=148, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:22:58,198 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148
2024-11-25T14:22:58,303 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:22:58,318 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=453 (was 409)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x19255325-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4eb56ec3-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6b949a6a
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x67e378ba-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x5fc5b243-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x17fd04ec-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2a17d2ee-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2a17d2ee-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0d3e603a-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x156d8469
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x42d4b10c-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x461585ca-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7dbbec35-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x35a043cb-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0d3e603a
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7dbbec35
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x35a043cb
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x110aae1f-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x461585ca
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x110aae1f
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x5fc5b243-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x42d4b10c-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x17fd04ec-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x461585ca-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x42d4b10c
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4eb56ec3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool.commonPool-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6b949a6a-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x156d8469-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x19255325
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x67e378ba-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x5fc5b243
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2a17d2ee
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x19255325-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x67e378ba
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x35a043cb-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4eb56ec3-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x156d8469-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0d3e603a-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7dbbec35-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x17fd04ec
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6b949a6a-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x110aae1f-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
 - Thread LEAK? -, OpenFileDescriptor=755 (was 700) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=466 (was 472), ProcessCount=11 (was 11), AvailableMemoryMB=5109 (was 5329)
2024-11-25T14:22:58,330 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=453, OpenFileDescriptor=755, MaxFileDescriptor=1048576, SystemLoadAverage=466, ProcessCount=11, AvailableMemoryMB=5108
2024-11-25T14:22:58,332 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:22:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:22:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149
2024-11-25T14:22:58,338 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:58,338 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7cd62759 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1bb8963d
2024-11-25T14:22:58,350 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x7cd62759 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7cd62759 to 127.0.0.1:51135
2024-11-25T14:22:58,350 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:58,375 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149
2024-11-25T14:22:58,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=150
2024-11-25T14:22:58,528 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=150}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:22:58,575 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=150}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:58,578 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=150}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=150
2024-11-25T14:22:58,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=150
2024-11-25T14:22:58,581 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:58,586 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149
2024-11-25T14:22:58,586 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 206 msec
2024-11-25T14:22:58,586 INFO  [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:22:58,595 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 254 msec
2024-11-25T14:22:58,599 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:58,600 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b302d59 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@21aac40f
2024-11-25T14:22:58,626 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:58,626 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149
2024-11-25T14:22:58,652 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:22:58,653 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151
2024-11-25T14:22:58,655 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:58,656 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x16d2818c to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72cb8eff
2024-11-25T14:22:58,666 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x16d2818c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x16d2818c to 127.0.0.1:51135
2024-11-25T14:22:58,666 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:58,684 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151
2024-11-25T14:22:58,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=152
2024-11-25T14:22:58,837 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:58,839 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:58,839 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:58,839 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:58,839 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:58,841 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b8c4161 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@14e8441f
2024-11-25T14:22:58,842 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:58,862 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:58,862 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:58,862 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=152
2024-11-25T14:22:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=152
2024-11-25T14:22:58,864 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:58,866 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151
2024-11-25T14:22:58,866 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 181 msec
2024-11-25T14:22:58,866 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:22:58,867 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 213 msec
2024-11-25T14:22:58,875 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:58,876 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x06c00e63 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7854ee2f
2024-11-25T14:22:58,883 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:58,883 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151
2024-11-25T14:22:58,973 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:58,974 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-11-25T14:22:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153
2024-11-25T14:22:58,978 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:58,979 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4354dce5 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4b8b3d75
2024-11-25T14:22:59,000 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x4354dce5 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4354dce5 to 127.0.0.1:51135
2024-11-25T14:22:59,000 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:59,065 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153
2024-11-25T14:22:59,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=154
2024-11-25T14:22:59,218 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:59,219 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:59,219 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:59,219 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:59,219 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:59,221 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x37ad6ee2 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@60518622
2024-11-25T14:22:59,221 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:59,248 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:59,249 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:59,249 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=154
2024-11-25T14:22:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=154
2024-11-25T14:22:59,252 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:59,254 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153
2024-11-25T14:22:59,254 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-11-25T14:22:59,254 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 187 msec
2024-11-25T14:22:59,255 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 280 msec
2024-11-25T14:22:59,264 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:59,264 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x09af5d06 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@282b230c
2024-11-25T14:22:59,275 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:59,275 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153
2024-11-25T14:22:59,292 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:59,293 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false
2024-11-25T14:22:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155
2024-11-25T14:22:59,295 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:59,296 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59e0d7e7 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@521f1e13
2024-11-25T14:22:59,308 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x59e0d7e7 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x59e0d7e7 to 127.0.0.1:51135
2024-11-25T14:22:59,308 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:59,309 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:59,310 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=155, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:59,311 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=155, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=17 msec
2024-11-25T14:22:59,311 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155
2024-11-25T14:22:59,402 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config
2024-11-25T14:22:59,403 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false
2024-11-25T14:22:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156
2024-11-25T14:22:59,406 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:59,407 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0403c6f5 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@436c2d9d
2024-11-25T14:22:59,416 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x0403c6f5 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0403c6f5 to 127.0.0.1:51135
2024-11-25T14:22:59,417 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:59,433 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156
2024-11-25T14:22:59,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=157
2024-11-25T14:22:59,586 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:22:59,587 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:22:59,587 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:22:59,588 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:22:59,588 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:59,589 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2600d56f to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@355ab1b1
2024-11-25T14:22:59,589 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:22:59,608 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:22:59,609 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:22:59,609 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157
2024-11-25T14:22:59,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=157
2024-11-25T14:22:59,612 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:22:59,614 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156
2024-11-25T14:22:59,614 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false
2024-11-25T14:22:59,615 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec
2024-11-25T14:22:59,616 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 212 msec
2024-11-25T14:22:59,626 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:59,627 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f8272d4 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@38645996
2024-11-25T14:22:59,633 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:22:59,633 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:22:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156
2024-11-25T14:22:59,722 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:22:59,723 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false
2024-11-25T14:22:59,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:22:59,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158
2024-11-25T14:22:59,727 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:22:59,728 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5ca3e10c to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@18402183
2024-11-25T14:22:59,742 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x5ca3e10c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5ca3e10c to 127.0.0.1:51135
2024-11-25T14:22:59,742 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:22:59,742 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:22:59,745 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=158, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:22:59,746 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=158, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=22 msec
2024-11-25T14:22:59,746 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:22:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158
2024-11-25T14:22:59,833 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config
2024-11-25T14:22:59,833 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:22:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=159, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:22:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159
2024-11-25T14:22:59,838 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:59,839 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:59,839 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:59,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:22:59,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:59,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:22:59,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:22:59,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:22:59,854 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:22:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159
2024-11-25T14:23:00,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=160
2024-11-25T14:23:00,009 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:23:00,009 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:23:00,009 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:23:00,009 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:00,009 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:00,011 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6957ab67 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@122444c1
2024-11-25T14:23:00,011 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:00,012 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:23:00,019 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=160
2024-11-25T14:23:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=160
2024-11-25T14:23:00,022 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:00,026 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159
2024-11-25T14:23:00,026 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:23:00,026 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 168 msec
2024-11-25T14:23:00,026 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:23:00,042 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:23:00,046 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 209 msec
2024-11-25T14:23:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159
2024-11-25T14:23:00,152 INFO  [ForkJoinPool.commonPool-worker-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:00,153 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=161, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=161
2024-11-25T14:23:00,155 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:00,157 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=161, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:00,158 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=161, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:23:00,158 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=161
2024-11-25T14:23:00,262 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:23:00,263 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:23:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162
2024-11-25T14:23:00,264 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:00,265 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=162, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:00,266 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=162, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec
2024-11-25T14:23:00,266 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162
2024-11-25T14:23:00,372 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:23:00,390 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=477 (was 453)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2600d56f
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x37ad6ee2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6957ab67
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x09af5d06-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1b8c4161-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1b8c4161
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x37ad6ee2-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x06c00e63-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2f8272d4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2600d56f-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2600d56f-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x09af5d06-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2f8272d4-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1b8c4161-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6b302d59-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2f8272d4-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6957ab67-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x37ad6ee2-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x06c00e63-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6b302d59-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6957ab67-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6b302d59
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x09af5d06
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x06c00e63
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=787 (was 755) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=466 (was 466), ProcessCount=11 (was 11), AvailableMemoryMB=4977 (was 5108)
2024-11-25T14:23:00,403 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=477, OpenFileDescriptor=787, MaxFileDescriptor=1048576, SystemLoadAverage=466, ProcessCount=11, AvailableMemoryMB=4976
2024-11-25T14:23:00,404 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:23:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=163, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:23:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=163
2024-11-25T14:23:00,407 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:00,408 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7b073e68 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@34eba5a7
2024-11-25T14:23:00,416 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x7b073e68 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7b073e68 to 127.0.0.1:51135
2024-11-25T14:23:00,417 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:00,451 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=163
2024-11-25T14:23:00,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=164
2024-11-25T14:23:00,603 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=164}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:23:00,642 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=164}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:00,645 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=164}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=164
2024-11-25T14:23:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=164
2024-11-25T14:23:00,647 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:00,649 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163
2024-11-25T14:23:00,649 INFO  [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:23:00,649 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 197 msec
2024-11-25T14:23:00,650 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 245 msec
2024-11-25T14:23:00,660 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:00,661 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f954016 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3884a733
2024-11-25T14:23:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=163
2024-11-25T14:23:00,722 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:00,723 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:23:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=165, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:23:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=165
2024-11-25T14:23:00,726 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:00,727 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5476a9e1 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@61c5059a
2024-11-25T14:23:00,783 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:00,783 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x5476a9e1 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5476a9e1 to 127.0.0.1:51135
2024-11-25T14:23:00,783 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:00,783 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:00,783 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:00,785 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=165, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:00,786 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=165, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=62 msec
2024-11-25T14:23:00,786 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=165
2024-11-25T14:23:00,833 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists
2024-11-25T14:23:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:23:00,834 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:23:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166
2024-11-25T14:23:00,837 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:00,838 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=166, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:00,840 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=166, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec
2024-11-25T14:23:00,840 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166
2024-11-25T14:23:00,943 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:23:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:23:00,944 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:23:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=167, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:23:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=167
2024-11-25T14:23:00,947 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test2'
2024-11-25T14:23:00,948 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7209bef9 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2d61e595
2024-11-25T14:23:00,958 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x7209bef9 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7209bef9 to 127.0.0.1:51135
2024-11-25T14:23:00,958 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test2
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:00,984 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=167
2024-11-25T14:23:01,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=168
2024-11-25T14:23:01,136 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=168}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=ADD_PEER
2024-11-25T14:23:01,174 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=168}] regionserver.ReplicationSource(231): queueId=2-cb69081f5976,36701,1732544544686, ReplicationSource: 2, currentBandwidth=0
2024-11-25T14:23:01,179 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=168}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=168
2024-11-25T14:23:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=168
2024-11-25T14:23:01,184 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:01,188 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167
2024-11-25T14:23:01,188 INFO  [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 2, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:23:01,188 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 200 msec
2024-11-25T14:23:01,191 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 244 msec
2024-11-25T14:23:01,201 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test2'
2024-11-25T14:23:01,203 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1ab36059 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4f11f785
2024-11-25T14:23:01,219 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:01,220 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=167
2024-11-25T14:23:01,263 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 2 completed
2024-11-25T14:23:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:23:01,267 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=169, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=169
2024-11-25T14:23:01,274 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:01,275 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:01,275 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:01,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:01,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:01,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:01,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:01,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:01,284 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=169
2024-11-25T14:23:01,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=170
2024-11-25T14:23:01,437 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:23:01,437 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:23:01,437 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:23:01,437 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:01,438 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:01,439 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x31355914 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@34fccf6c
2024-11-25T14:23:01,439 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:01,440 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:23:01,446 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=170
2024-11-25T14:23:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=170
2024-11-25T14:23:01,449 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:01,452 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169
2024-11-25T14:23:01,452 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:23:01,452 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:23:01,452 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 165 msec
2024-11-25T14:23:01,464 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:23:01,467 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 197 msec
2024-11-25T14:23:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=169
2024-11-25T14:23:01,592 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:23:01,594 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:23:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=171, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=171
2024-11-25T14:23:01,597 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/sync-rep-state
2024-11-25T14:23:01,598 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state
2024-11-25T14:23:01,598 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/new-sync-rep-state
2024-11-25T14:23:01,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/new-sync-rep-state
2024-11-25T14:23:01,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2
2024-11-25T14:23:01,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state
2024-11-25T14:23:01,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/sync-rep-state
2024-11-25T14:23:01,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2
2024-11-25T14:23:01,609 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=171
2024-11-25T14:23:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22
2024-11-25T14:23:01,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=172
2024-11-25T14:23:01,762 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=REMOVE_PEER
2024-11-25T14:23:01,762 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 2: 0
2024-11-25T14:23:01,762 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSource(687): peerId=2, Closing source 2-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:23:01,762 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=2, Interrupted while sleeping between retries
2024-11-25T14:23:01,762 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test2'
2024-11-25T14:23:01,764 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x220c9081 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@197c9efa
2024-11-25T14:23:01,765 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,2-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:01,765 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(562): Done with the queue 2-cb69081f5976,36701,1732544544686
2024-11-25T14:23:01,768 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=172
2024-11-25T14:23:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=172
2024-11-25T14:23:01,771 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:01,775 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 2, move on
2024-11-25T14:23:01,775 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 2, move on
2024-11-25T14:23:01,776 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171
2024-11-25T14:23:01,776 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 162 msec
2024-11-25T14:23:01,791 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 2
2024-11-25T14:23:01,796 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 197 msec
2024-11-25T14:23:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=171
2024-11-25T14:23:01,913 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed
2024-11-25T14:23:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:23:01,915 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=173, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=173
2024-11-25T14:23:01,918 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:01,928 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=173, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:01,930 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=173, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=14 msec
2024-11-25T14:23:01,930 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=173
2024-11-25T14:23:02,023 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:23:02,024 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:23:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:02,029 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:02,034 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=174, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:02,036 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=174, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=10 msec
2024-11-25T14:23:02,036 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174
2024-11-25T14:23:02,040 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:23:02,066 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=489 (was 477)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x220c9081-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1ab36059-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2f954016-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x220c9081-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1ab36059-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1ab36059
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2f954016-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2f954016
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x220c9081
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x31355914
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x31355914-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x31355914-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
 - Thread LEAK? -, OpenFileDescriptor=803 (was 787) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=466 (was 466), ProcessCount=11 (was 11), AvailableMemoryMB=4910 (was 4976)
2024-11-25T14:23:02,080 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=489, OpenFileDescriptor=803, MaxFileDescriptor=1048576, SystemLoadAverage=466, ProcessCount=11, AvailableMemoryMB=4908
2024-11-25T14:23:02,081 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:23:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=175, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:23:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=175
2024-11-25T14:23:02,086 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: java.lang.ClassNotFoundException: whatever
	at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?]
	at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?]
	at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?]
	at java.lang.Class.forName0(Native Method) ~[?:?]
	at java.lang.Class.forName(Class.java:375) ~[?:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?]
	... 11 more
2024-11-25T14:23:02,088 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=175, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
Caused by: java.lang.ClassNotFoundException: whatever
	at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?]
	at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?]
	at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?]
	at java.lang.Class.forName0(Native Method) ~[?:?]
	at java.lang.Class.forName(Class.java:375) ~[?:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:02,089 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=175, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=6 msec
2024-11-25T14:23:02,089 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=175
2024-11-25T14:23:02,192 INFO  [ForkJoinPool.commonPool-worker-1 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever
2024-11-25T14:23:02,193 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176
2024-11-25T14:23:02,195 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:02,196 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=176, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:02,197 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=176, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-25T14:23:02,197 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176
2024-11-25T14:23:02,302 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:23:02,303 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:23:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=177, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=177
2024-11-25T14:23:02,305 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:02,305 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=177, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:02,306 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=177, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-25T14:23:02,306 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=177
2024-11-25T14:23:02,412 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:23:02,430 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=489 (was 489), OpenFileDescriptor=803 (was 803), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=466 (was 466), ProcessCount=11 (was 11), AvailableMemoryMB=4892 (was 4908)
2024-11-25T14:23:02,444 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=489, OpenFileDescriptor=803, MaxFileDescriptor=1048576, SystemLoadAverage=466, ProcessCount=11, AvailableMemoryMB=4889
2024-11-25T14:23:02,445 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:23:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:23:02,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178
2024-11-25T14:23:02,448 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?]
	... 12 more
2024-11-25T14:23:02,449 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=178, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:02,450 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=178, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=4 msec
2024-11-25T14:23:02,450 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178
2024-11-25T14:23:02,552 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever
2024-11-25T14:23:02,553 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=179, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=179
2024-11-25T14:23:02,555 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:02,556 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=179, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:02,557 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=179, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-25T14:23:02,557 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=179
2024-11-25T14:23:02,662 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:23:02,663 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:23:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180
2024-11-25T14:23:02,665 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:02,666 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=180, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:02,668 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=180, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:23:02,668 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180
2024-11-25T14:23:02,772 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:23:02,790 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=489 (was 489), OpenFileDescriptor=803 (was 803), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=468 (was 466) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4857 (was 4889)
2024-11-25T14:23:02,801 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=489, OpenFileDescriptor=803, MaxFileDescriptor=1048576, SystemLoadAverage=468, ProcessCount=11, AvailableMemoryMB=4856
2024-11-25T14:23:02,803 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:23:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=181, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:23:02,808 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:02,809 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3371c7b2 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3b6c9e56
2024-11-25T14:23:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=181
2024-11-25T14:23:02,834 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x3371c7b2 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3371c7b2 to 127.0.0.1:51135
2024-11-25T14:23:02,834 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:02,911 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=182, ppid=181, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=181
2024-11-25T14:23:03,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=182
2024-11-25T14:23:03,065 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=182}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:23:03,101 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=182}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:03,105 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=182}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=182
2024-11-25T14:23:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=182
2024-11-25T14:23:03,108 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:03,110 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=182, resume processing ppid=181
2024-11-25T14:23:03,110 INFO  [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:23:03,110 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, ppid=181, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 197 msec
2024-11-25T14:23:03,111 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 307 msec
2024-11-25T14:23:03,126 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:03,127 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4385702e to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@ec6a5b6
2024-11-25T14:23:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=181
2024-11-25T14:23:03,136 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:03,141 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:23:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=183, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:23:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=183
2024-11-25T14:23:03,150 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:03,151 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2c05a57f to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@26613a96
2024-11-25T14:23:03,152 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:03,152 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:03,171 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x2c05a57f {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2c05a57f to 127.0.0.1:51135
2024-11-25T14:23:03,172 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:03,196 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=184, ppid=183, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=183
2024-11-25T14:23:03,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=184
2024-11-25T14:23:03,349 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:23:03,351 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:23:03,351 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:23:03,351 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:03,351 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:03,353 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7c998e94 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2847ce7e
2024-11-25T14:23:03,353 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:03,369 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:03,369 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:23:03,370 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=184
2024-11-25T14:23:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=184
2024-11-25T14:23:03,373 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:03,375 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=184, resume processing ppid=183
2024-11-25T14:23:03,375 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:23:03,375 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=184, ppid=183, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec
2024-11-25T14:23:03,376 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=183, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 234 msec
2024-11-25T14:23:03,384 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:03,385 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1d7901e1 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5af00a25
2024-11-25T14:23:03,391 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:03,391 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=183
2024-11-25T14:23:03,462 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:23:03,463 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false
2024-11-25T14:23:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=185, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:23:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=185
2024-11-25T14:23:03,465 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:03,466 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x70385e13 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@23b4b1bb
2024-11-25T14:23:03,500 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x70385e13 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x70385e13 to 127.0.0.1:51135
2024-11-25T14:23:03,500 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:03,552 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-25T14:23:03,560 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=186, ppid=185, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=185
2024-11-25T14:23:03,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=186
2024-11-25T14:23:03,714 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:23:03,715 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:23:03,715 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:23:03,715 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:03,715 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:03,717 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x576ce492 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e26033b
2024-11-25T14:23:03,717 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:03,733 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:03,733 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:23:03,733 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=186
2024-11-25T14:23:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=186
2024-11-25T14:23:03,736 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:03,737 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=186, resume processing ppid=185
2024-11-25T14:23:03,738 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false
2024-11-25T14:23:03,738 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=186, ppid=185, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec
2024-11-25T14:23:03,739 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=185, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 275 msec
2024-11-25T14:23:03,749 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:03,750 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x473e126c to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@438ecd13
2024-11-25T14:23:03,774 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:03,775 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=185
2024-11-25T14:23:03,782 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:23:03,783 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:03,784 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-11-25T14:23:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=187, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:23:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=187
2024-11-25T14:23:03,788 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:03,789 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1dbcc6a7 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@74d8c047
2024-11-25T14:23:03,799 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x1dbcc6a7 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1dbcc6a7 to 127.0.0.1:51135
2024-11-25T14:23:03,800 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:03,817 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=188, ppid=187, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=187
2024-11-25T14:23:03,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=188
2024-11-25T14:23:03,969 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:23:03,970 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:23:03,970 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:23:03,970 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:03,970 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:03,972 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0cfe6fde to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@be7f87c
2024-11-25T14:23:03,972 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:03,988 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:03,989 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:23:03,989 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=188
2024-11-25T14:23:03,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=188
2024-11-25T14:23:03,992 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:03,994 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=188, resume processing ppid=187
2024-11-25T14:23:03,994 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false
2024-11-25T14:23:03,994 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=188, ppid=187, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 175 msec
2024-11-25T14:23:03,997 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=187, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 210 msec
2024-11-25T14:23:04,004 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:04,005 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7df66477 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3efad0b
2024-11-25T14:23:04,016 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:04,016 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=187
2024-11-25T14:23:04,102 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:23:04,103 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:04,104 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=189, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=189
2024-11-25T14:23:04,109 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:04,109 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:04,110 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:04,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:04,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:04,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:04,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:04,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:04,117 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=190, ppid=189, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=189
2024-11-25T14:23:04,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=190
2024-11-25T14:23:04,271 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:23:04,271 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:23:04,271 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:23:04,271 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:04,271 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:04,273 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6eec713e to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7eb0298a
2024-11-25T14:23:04,274 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:04,274 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:23:04,277 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=190
2024-11-25T14:23:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=190
2024-11-25T14:23:04,281 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:04,284 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=190, resume processing ppid=189
2024-11-25T14:23:04,284 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:23:04,285 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=190, ppid=189, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 164 msec
2024-11-25T14:23:04,285 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:23:04,291 INFO  [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:23:04,293 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=189, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 187 msec
2024-11-25T14:23:04,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=189
2024-11-25T14:23:04,422 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:04,423 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:04,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=191, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=191
2024-11-25T14:23:04,425 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:04,427 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=191, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:04,428 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=191, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:23:04,428 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=191
2024-11-25T14:23:04,532 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:23:04,533 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:23:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=192, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=192
2024-11-25T14:23:04,535 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:04,536 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=192, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:04,537 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=192, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-25T14:23:04,537 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=192
2024-11-25T14:23:04,643 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:23:04,663 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=511 (was 489)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4385702e
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x473e126c-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0cfe6fde-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6eec713e-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7df66477-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x473e126c-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7c998e94
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1d7901e1-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4385702e-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7df66477-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1d7901e1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6eec713e-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x473e126c
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x576ce492-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7c998e94-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7df66477
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0cfe6fde-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x4385702e-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6eec713e
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x1d7901e1-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x576ce492-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7c998e94-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0cfe6fde
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x576ce492
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=831 (was 803) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=468 (was 468), ProcessCount=11 (was 11), AvailableMemoryMB=4656 (was 4856)
2024-11-25T14:23:04,666 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=511 is superior to 500
2024-11-25T14:23:04,680 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=511, OpenFileDescriptor=831, MaxFileDescriptor=1048576, SystemLoadAverage=468, ProcessCount=11, AvailableMemoryMB=4653
2024-11-25T14:23:04,681 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=511 is superior to 500
2024-11-25T14:23:04,681 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:23:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=193, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:23:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=193
2024-11-25T14:23:04,685 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:04,686 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x560391aa to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@49ec0f7c
2024-11-25T14:23:04,730 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x560391aa {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x560391aa to 127.0.0.1:51135
2024-11-25T14:23:04,730 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=193
2024-11-25T14:23:04,809 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=194, ppid=193, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:04,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=194
2024-11-25T14:23:04,961 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=194}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:23:04,991 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=194}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:04,994 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=194}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=194
2024-11-25T14:23:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=194
2024-11-25T14:23:04,997 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:04,998 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=194, resume processing ppid=193
2024-11-25T14:23:04,998 INFO  [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:23:04,999 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=194, ppid=193, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec
2024-11-25T14:23:05,000 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=193, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 318 msec
2024-11-25T14:23:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=193
2024-11-25T14:23:05,002 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:05,003 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:23:05,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=195, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:23:05,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=195
2024-11-25T14:23:05,005 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:05,006 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2dc6d791 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@67818a4c
2024-11-25T14:23:05,010 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:05,010 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3d388757 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@325fc926
2024-11-25T14:23:05,024 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x2dc6d791 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2dc6d791 to 127.0.0.1:51135
2024-11-25T14:23:05,024 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:05,024 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:05,025 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:05,042 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=196, ppid=195, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=195
2024-11-25T14:23:05,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=196
2024-11-25T14:23:05,194 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:23:05,195 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:23:05,195 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:23:05,195 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:05,195 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:05,196 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a91c282 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@327e4a08
2024-11-25T14:23:05,196 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:05,214 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:05,214 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:23:05,214 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=196
2024-11-25T14:23:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=196
2024-11-25T14:23:05,217 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:05,218 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=196, resume processing ppid=195
2024-11-25T14:23:05,218 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false
2024-11-25T14:23:05,218 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=196, ppid=195, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 175 msec
2024-11-25T14:23:05,219 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=195, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 215 msec
2024-11-25T14:23:05,229 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:05,229 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x610209d0 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@47759b97
2024-11-25T14:23:05,241 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:05,241 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=195
2024-11-25T14:23:05,322 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:23:05,322 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:05,323 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false
2024-11-25T14:23:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=197, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:23:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=197
2024-11-25T14:23:05,325 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:05,326 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x01403537 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@65fcc148
2024-11-25T14:23:05,333 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x01403537 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x01403537 to 127.0.0.1:51135
2024-11-25T14:23:05,333 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:05,350 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=198, ppid=197, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=197
2024-11-25T14:23:05,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=198
2024-11-25T14:23:05,503 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:23:05,504 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:23:05,504 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:23:05,504 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:05,504 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:05,506 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x25adfca7 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1a734a7
2024-11-25T14:23:05,506 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:05,526 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:05,526 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:23:05,526 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=198
2024-11-25T14:23:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=198
2024-11-25T14:23:05,529 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:05,539 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=198, resume processing ppid=197
2024-11-25T14:23:05,539 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=198, ppid=197, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec
2024-11-25T14:23:05,540 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false
2024-11-25T14:23:05,541 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=197, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 217 msec
2024-11-25T14:23:05,546 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:05,548 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x030385ba to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4ad275cc
2024-11-25T14:23:05,569 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:05,569 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=197
2024-11-25T14:23:05,642 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:23:05,643 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:05,644 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:05,645 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-11-25T14:23:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=199, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:23:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=199
2024-11-25T14:23:05,648 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:05,649 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x63bb3e23 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@50d9968e
2024-11-25T14:23:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=199
2024-11-25T14:23:05,763 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x63bb3e23 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x63bb3e23 to 127.0.0.1:51135
2024-11-25T14:23:05,763 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:05,892 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=200, ppid=199, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=199
2024-11-25T14:23:06,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=200
2024-11-25T14:23:06,045 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:23:06,046 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:23:06,046 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:23:06,046 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:06,046 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:06,048 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x39819a4b to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@76650b54
2024-11-25T14:23:06,048 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:06,064 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:06,064 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:23:06,064 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=200
2024-11-25T14:23:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=200
2024-11-25T14:23:06,067 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:06,068 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=200, resume processing ppid=199
2024-11-25T14:23:06,068 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=200, ppid=199, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 175 msec
2024-11-25T14:23:06,068 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-11-25T14:23:06,070 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=199, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 423 msec
2024-11-25T14:23:06,079 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:06,080 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x11119392 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5752dbe2
2024-11-25T14:23:06,117 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:06,117 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=199
2024-11-25T14:23:06,272 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:23:06,272 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:06,273 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:06,274 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-11-25T14:23:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=201, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:23:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=201
2024-11-25T14:23:06,276 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:06,277 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x409655c1 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@631b8c22
2024-11-25T14:23:06,336 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x409655c1 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x409655c1 to 127.0.0.1:51135
2024-11-25T14:23:06,336 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=201
2024-11-25T14:23:06,452 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=202, ppid=201, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=201
2024-11-25T14:23:06,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=202
2024-11-25T14:23:06,604 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:23:06,605 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:23:06,606 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:23:06,606 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:06,606 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:06,607 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3f3626c0 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@73200321
2024-11-25T14:23:06,607 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:06,630 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:06,630 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:23:06,630 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=202
2024-11-25T14:23:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=202
2024-11-25T14:23:06,633 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:06,635 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=202, resume processing ppid=201
2024-11-25T14:23:06,635 INFO  [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false
2024-11-25T14:23:06,635 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=202, ppid=201, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 181 msec
2024-11-25T14:23:06,636 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=201, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 361 msec
2024-11-25T14:23:06,645 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:06,646 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x74e9f1c5 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4fa0bd0
2024-11-25T14:23:06,658 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:06,658 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=201
2024-11-25T14:23:06,902 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:23:06,903 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:06,903 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:06,904 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-25T14:23:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=203, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:23:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=203
2024-11-25T14:23:06,907 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:06,907 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5aa0db97 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2cd44c88
2024-11-25T14:23:06,925 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x5aa0db97 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5aa0db97 to 127.0.0.1:51135
2024-11-25T14:23:06,926 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:06,942 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=204, ppid=203, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=203
2024-11-25T14:23:07,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=204
2024-11-25T14:23:07,094 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:23:07,096 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:23:07,096 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:23:07,096 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:07,096 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:07,097 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x34bd6e27 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1e17f0fe
2024-11-25T14:23:07,098 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:07,120 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:07,120 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:23:07,120 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=204
2024-11-25T14:23:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=204
2024-11-25T14:23:07,123 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:07,124 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=204, resume processing ppid=203
2024-11-25T14:23:07,124 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-25T14:23:07,125 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=204, ppid=203, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 181 msec
2024-11-25T14:23:07,126 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=203, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 221 msec
2024-11-25T14:23:07,140 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:07,142 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6ab8c4bc to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e473a90
2024-11-25T14:23:07,152 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:07,152 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=203
2024-11-25T14:23:07,222 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:23:07,223 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:07,224 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:07,225 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-25T14:23:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=205, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:23:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=205
2024-11-25T14:23:07,229 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:07,230 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1d5cb6a3 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@43329c0d
2024-11-25T14:23:07,241 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x1d5cb6a3 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1d5cb6a3 to 127.0.0.1:51135
2024-11-25T14:23:07,241 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:07,258 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=206, ppid=205, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=205
2024-11-25T14:23:07,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=206
2024-11-25T14:23:07,413 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:23:07,415 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:23:07,415 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:23:07,415 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:07,415 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:07,418 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x762f3bd7 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6d969476
2024-11-25T14:23:07,418 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:07,437 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:07,437 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:23:07,438 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=206
2024-11-25T14:23:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=206
2024-11-25T14:23:07,440 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:07,442 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=206, resume processing ppid=205
2024-11-25T14:23:07,442 INFO  [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-25T14:23:07,442 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=206, ppid=205, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 182 msec
2024-11-25T14:23:07,444 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=205, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 217 msec
2024-11-25T14:23:07,453 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:07,454 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x723f432e to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@112098ca
2024-11-25T14:23:07,465 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:07,465 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=205
2024-11-25T14:23:07,542 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:23:07,543 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:07,545 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-25T14:23:07,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=207, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:23:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=207
2024-11-25T14:23:07,549 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:07,551 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x403e78c3 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4f8691bc
2024-11-25T14:23:07,558 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x403e78c3 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x403e78c3 to 127.0.0.1:51135
2024-11-25T14:23:07,558 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:07,575 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=208, ppid=207, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=207
2024-11-25T14:23:07,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=208
2024-11-25T14:23:07,727 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:23:07,728 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=208
2024-11-25T14:23:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=208
2024-11-25T14:23:07,731 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:07,732 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=208, resume processing ppid=207
2024-11-25T14:23:07,732 INFO  [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false
2024-11-25T14:23:07,732 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=208, ppid=207, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 156 msec
2024-11-25T14:23:07,734 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=207, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 187 msec
2024-11-25T14:23:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=207
2024-11-25T14:23:07,863 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:23:07,863 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:07,864 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:07,865 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false
2024-11-25T14:23:07,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=209, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:23:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=209
2024-11-25T14:23:07,867 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:07,868 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7812c526 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@52d8dedd
2024-11-25T14:23:07,874 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x7812c526 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7812c526 to 127.0.0.1:51135
2024-11-25T14:23:07,874 WARN  [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:07,892 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=210, ppid=209, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:07,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=209
2024-11-25T14:23:08,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=210
2024-11-25T14:23:08,044 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:23:08,045 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:23:08,045 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:23:08,045 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:08,045 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:08,047 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x68272c06 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@54e598f
2024-11-25T14:23:08,047 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:08,062 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:08,063 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:23:08,063 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=210
2024-11-25T14:23:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=210
2024-11-25T14:23:08,065 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:08,068 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=210, resume processing ppid=209
2024-11-25T14:23:08,068 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false
2024-11-25T14:23:08,068 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=210, ppid=209, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 174 msec
2024-11-25T14:23:08,070 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=209, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 203 msec
2024-11-25T14:23:08,081 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:08,082 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0018a0f2 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2ed6a9ee
2024-11-25T14:23:08,099 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:08,100 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:08,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=209
2024-11-25T14:23:08,182 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:23:08,183 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:08,184 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false
2024-11-25T14:23:08,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=211, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:23:08,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=211
2024-11-25T14:23:08,186 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:08,187 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x00409ad7 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@14c15900
2024-11-25T14:23:08,198 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x00409ad7 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x00409ad7 to 127.0.0.1:51135
2024-11-25T14:23:08,198 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:08,217 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=212, ppid=211, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=211
2024-11-25T14:23:08,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=212
2024-11-25T14:23:08,369 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:23:08,370 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:23:08,370 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:23:08,370 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:08,370 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:08,371 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x17b1fd01 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4e465d1a
2024-11-25T14:23:08,372 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:08,387 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:08,387 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:23:08,387 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=212
2024-11-25T14:23:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=212
2024-11-25T14:23:08,389 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:08,391 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=212, resume processing ppid=211
2024-11-25T14:23:08,391 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=212, ppid=211, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 173 msec
2024-11-25T14:23:08,391 INFO  [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false
2024-11-25T14:23:08,393 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=211, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 207 msec
2024-11-25T14:23:08,401 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:08,401 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6411cc8e to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6c3bf257
2024-11-25T14:23:08,448 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:08,448 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=211
2024-11-25T14:23:08,502 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:23:08,503 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:08,503 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:08,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=213, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=213
2024-11-25T14:23:08,507 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:08,507 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:08,507 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:08,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:08,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:08,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:08,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:08,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:08,517 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=214, ppid=213, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=213
2024-11-25T14:23:08,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=214
2024-11-25T14:23:08,669 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:23:08,669 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:23:08,669 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:23:08,669 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:08,669 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:08,671 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6d5b8f16 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@51e7c
2024-11-25T14:23:08,671 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:08,671 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:23:08,673 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=214
2024-11-25T14:23:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=214
2024-11-25T14:23:08,676 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:08,677 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=214, resume processing ppid=213
2024-11-25T14:23:08,677 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:23:08,678 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=214, ppid=213, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec
2024-11-25T14:23:08,678 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:23:08,683 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:23:08,685 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=213, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 180 msec
2024-11-25T14:23:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=213
2024-11-25T14:23:08,822 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:08,823 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=215, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=215
2024-11-25T14:23:08,824 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:08,825 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=215, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:08,826 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=215, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec
2024-11-25T14:23:08,826 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=215
2024-11-25T14:23:08,932 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:23:08,933 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:23:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=216, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=216
2024-11-25T14:23:08,935 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:08,936 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=216, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:08,937 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=216, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-25T14:23:08,937 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=216
2024-11-25T14:23:09,043 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:23:09,065 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=563 (was 511)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x030385ba-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x17b1fd01-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6d5b8f16-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x39819a4b-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x25adfca7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x723f432e-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x723f432e-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7a91c282-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x74e9f1c5-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x3f3626c0-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x74e9f1c5-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x762f3bd7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6d5b8f16-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x610209d0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x34bd6e27-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x68272c06-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x17b1fd01
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x610209d0-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x74e9f1c5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x762f3bd7-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x11119392
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0018a0f2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6d5b8f16
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x17b1fd01-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x39819a4b
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6ab8c4bc
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x3d388757
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6ab8c4bc-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x39819a4b-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6411cc8e-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7a91c282-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x25adfca7-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x68272c06-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6411cc8e
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x34bd6e27
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0018a0f2-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x030385ba
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x3f3626c0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x723f432e
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x030385ba-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x3d388757-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x0018a0f2-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x762f3bd7-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x610209d0-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x3f3626c0-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6ab8c4bc-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x6411cc8e-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x34bd6e27-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x3d388757-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x68272c06
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x11119392-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x25adfca7-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7a91c282
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x11119392-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
 - Thread LEAK? -, OpenFileDescriptor=901 (was 831) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=439 (was 468), ProcessCount=11 (was 11), AvailableMemoryMB=4258 (was 4653)
2024-11-25T14:23:09,065 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=563 is superior to 500
2024-11-25T14:23:09,077 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=563, OpenFileDescriptor=901, MaxFileDescriptor=1048576, SystemLoadAverage=439, ProcessCount=11, AvailableMemoryMB=4256
2024-11-25T14:23:09,077 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=563 is superior to 500
2024-11-25T14:23:09,078 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:23:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=217, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:23:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=217
2024-11-25T14:23:09,080 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:09,081 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1288abc5 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@59ef2634
2024-11-25T14:23:09,091 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x1288abc5 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1288abc5 to 127.0.0.1:51135
2024-11-25T14:23:09,091 WARN  [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:09,141 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=218, ppid=217, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=217
2024-11-25T14:23:09,288 INFO  [master/cb69081f5976:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-25T14:23:09,288 INFO  [master/cb69081f5976:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-25T14:23:09,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=218
2024-11-25T14:23:09,293 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=218}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:23:09,330 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=218}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:09,333 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=218}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=218
2024-11-25T14:23:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=218
2024-11-25T14:23:09,335 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:09,336 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=218, resume processing ppid=217
2024-11-25T14:23:09,336 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=218, ppid=217, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 194 msec
2024-11-25T14:23:09,336 INFO  [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:23:09,337 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=217, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 258 msec
2024-11-25T14:23:09,351 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:09,353 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ffe62d5 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@715804e6
2024-11-25T14:23:09,366 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:09,366 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=217
2024-11-25T14:23:09,392 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:23:09,393 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3992): Client=jenkins//172.17.0.3 disable replication peer, id=1
2024-11-25T14:23:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=219, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure
2024-11-25T14:23:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=219
2024-11-25T14:23:09,407 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=220, ppid=219, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=219
2024-11-25T14:23:09,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=220
2024-11-25T14:23:09,560 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=DISABLE_PEER
2024-11-25T14:23:09,561 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1
2024-11-25T14:23:09,561 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Peer 1 state or config changed. Will close the previous replication source and open a new one
2024-11-25T14:23:09,561 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:09,561 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:09,562 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2968e104 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@58d7f97a
2024-11-25T14:23:09,562 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:09,584 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:09,585 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSourceManager(500): Startup replication source for 1
2024-11-25T14:23:09,585 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=220
2024-11-25T14:23:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=220
2024-11-25T14:23:09,587 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for DISABLE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:09,589 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=220, resume processing ppid=219
2024-11-25T14:23:09,589 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=220, ppid=219, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 180 msec
2024-11-25T14:23:09,589 INFO  [PEWorker-1 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1
2024-11-25T14:23:09,590 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=219, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 196 msec
2024-11-25T14:23:09,601 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:09,602 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x54ad8515 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1195359d
2024-11-25T14:23:09,616 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:09,616 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=219
2024-11-25T14:23:09,712 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:23:09,713 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=221, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=221
2024-11-25T14:23:09,717 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:09,717 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:09,718 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:09,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:09,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:09,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:09,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:09,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:09,725 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=222, ppid=221, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:09,776 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-25T14:23:09,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=221
2024-11-25T14:23:09,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=222
2024-11-25T14:23:09,878 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:23:09,878 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:23:09,878 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:23:09,878 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:09,878 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:09,880 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x509f82b1 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@10064e70
2024-11-25T14:23:09,880 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:09,880 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:23:09,884 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=222
2024-11-25T14:23:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=222
2024-11-25T14:23:09,886 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:09,888 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=222, resume processing ppid=221
2024-11-25T14:23:09,888 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:23:09,888 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=222, ppid=221, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 161 msec
2024-11-25T14:23:09,888 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:23:09,896 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:23:09,898 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=221, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 183 msec
2024-11-25T14:23:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=221
2024-11-25T14:23:10,032 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:10,032 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=223, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=223
2024-11-25T14:23:10,035 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:10,036 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=223, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:10,037 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=223, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:23:10,037 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=223
2024-11-25T14:23:10,142 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:23:10,143 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:23:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=224, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:10,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=224
2024-11-25T14:23:10,145 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:10,147 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=224, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:10,148 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=224, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:23:10,148 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=224
2024-11-25T14:23:10,252 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:23:10,269 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=575 (was 563)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2968e104-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7ffe62d5-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x54ad8515-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2968e104-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x509f82b1-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7ffe62d5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x54ad8515
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x509f82b1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2968e104
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7ffe62d5-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x54ad8515-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x509f82b1-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
 - Thread LEAK? -, OpenFileDescriptor=917 (was 901) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=439 (was 439), ProcessCount=11 (was 11), AvailableMemoryMB=4189 (was 4256)
2024-11-25T14:23:10,269 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=575 is superior to 500
2024-11-25T14:23:10,281 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=575, OpenFileDescriptor=917, MaxFileDescriptor=1048576, SystemLoadAverage=439, ProcessCount=11, AvailableMemoryMB=4188
2024-11-25T14:23:10,281 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=575 is superior to 500
2024-11-25T14:23:10,282 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=dummy_peer
2024-11-25T14:23:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=225, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=225
2024-11-25T14:23:10,283 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:10,284 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=225, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:10,285 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=225, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec
2024-11-25T14:23:10,285 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=225
2024-11-25T14:23:10,392 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer
2024-11-25T14:23:10,392 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=226, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=226
2024-11-25T14:23:10,394 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:10,395 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=226, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:10,396 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=226, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-25T14:23:10,396 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=226
2024-11-25T14:23:10,502 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:23:10,502 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:23:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=227, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=227
2024-11-25T14:23:10,504 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:10,506 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=227, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:10,507 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=227, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:23:10,507 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=227
2024-11-25T14:23:10,612 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:23:10,631 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=575 (was 575), OpenFileDescriptor=917 (was 917), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=439 (was 439), ProcessCount=11 (was 11), AvailableMemoryMB=4168 (was 4188)
2024-11-25T14:23:10,631 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=575 is superior to 500
2024-11-25T14:23:10,645 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=575, OpenFileDescriptor=917, MaxFileDescriptor=1048576, SystemLoadAverage=439, ProcessCount=11, AvailableMemoryMB=4168
2024-11-25T14:23:10,645 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=575 is superior to 500
2024-11-25T14:23:10,646 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:23:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=228, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:23:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=228
2024-11-25T14:23:10,649 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:10,650 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x68b52354 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1ff75ce9
2024-11-25T14:23:10,657 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x68b52354 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x68b52354 to 127.0.0.1:51135
2024-11-25T14:23:10,658 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:10,683 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=229, ppid=228, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=228
2024-11-25T14:23:10,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=229
2024-11-25T14:23:10,837 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=229}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:23:10,874 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=229}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:10,876 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=229}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=229
2024-11-25T14:23:10,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=229
2024-11-25T14:23:10,879 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:10,880 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=229, resume processing ppid=228
2024-11-25T14:23:10,880 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=229, ppid=228, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 196 msec
2024-11-25T14:23:10,880 INFO  [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:23:10,882 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=228, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 234 msec
2024-11-25T14:23:10,895 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:10,896 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x26c26b7a to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4faafe3f
2024-11-25T14:23:10,908 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:10,908 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=228
2024-11-25T14:23:10,962 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null
2024-11-25T14:23:10,963 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=230, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=230
2024-11-25T14:23:10,966 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:10,967 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:10,967 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:10,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:10,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:10,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:10,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:10,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:10,976 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=231, ppid=230, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=230
2024-11-25T14:23:11,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=231
2024-11-25T14:23:11,130 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:23:11,130 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:23:11,130 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:23:11,130 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:11,131 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:11,134 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x762bb38c to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@45ff8bf6
2024-11-25T14:23:11,135 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:11,135 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:23:11,139 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=231
2024-11-25T14:23:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=231
2024-11-25T14:23:11,142 INFO  [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:11,145 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=231, resume processing ppid=230
2024-11-25T14:23:11,145 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:23:11,145 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=231, ppid=230, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 166 msec
2024-11-25T14:23:11,145 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:23:11,151 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:23:11,152 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=230, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 188 msec
2024-11-25T14:23:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=230
2024-11-25T14:23:11,281 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:11,282 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=232, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=232
2024-11-25T14:23:11,284 WARN  [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:11,285 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=232, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:11,286 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=232, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-25T14:23:11,286 WARN  [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=232
2024-11-25T14:23:11,392 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:23:11,392 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:23:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=233, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=233
2024-11-25T14:23:11,394 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:11,395 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=233, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:11,396 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=233, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-25T14:23:11,396 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=233
2024-11-25T14:23:11,502 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:23:11,520 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=581 (was 575)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x762bb38c-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x762bb38c
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x26c26b7a-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x26c26b7a
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x26c26b7a-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x762bb38c-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
 - Thread LEAK? -, OpenFileDescriptor=925 (was 917) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=439 (was 439), ProcessCount=11 (was 11), AvailableMemoryMB=4128 (was 4168)
2024-11-25T14:23:11,520 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=581 is superior to 500
2024-11-25T14:23:11,535 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=581, OpenFileDescriptor=925, MaxFileDescriptor=1048576, SystemLoadAverage=439, ProcessCount=11, AvailableMemoryMB=4127
2024-11-25T14:23:11,535 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=581 is superior to 500
2024-11-25T14:23:11,539 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:23:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] ipc.MetricsHBaseServer(152): Unknown exception type
java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] ipc.CallRunner(138): callId: 695 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.3:37128 deadline: 1732544651539, exception=java.io.IOException: Replication peer modification disabled
2024-11-25T14:23:11,540 WARN  [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 1 ms
java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?]
	at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 32 more
2024-11-25T14:23:11,652 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:23:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] ipc.MetricsHBaseServer(152): Unknown exception type
java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] ipc.CallRunner(138): callId: 696 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.3:37128 deadline: 1732544651652, exception=java.io.IOException: Replication peer modification disabled
2024-11-25T14:23:11,653 WARN  [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 114 ms
java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?]
	at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 32 more
2024-11-25T14:23:11,653 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions:
2024-11-25T14:23:11.540Z, java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-25T14:23:11.653Z, java.io.IOException: java.io.IOException: Replication peer modification disabled
	at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963)
	at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975)
	at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)


2024-11-25T14:23:11,656 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=234, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=234
2024-11-25T14:23:11,659 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:11,661 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=234, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:11,662 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=234, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec
2024-11-25T14:23:11,662 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=234
2024-11-25T14:23:11,762 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:23:11,763 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:23:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=235, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=235
2024-11-25T14:23:11,765 WARN  [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:11,766 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=235, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:11,767 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=235, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-25T14:23:11,767 WARN  [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22
2024-11-25T14:23:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=235
2024-11-25T14:23:11,873 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:23:11,893 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=581 (was 581), OpenFileDescriptor=925 (was 925), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=439 (was 439), ProcessCount=11 (was 11), AvailableMemoryMB=4093 (was 4127)
2024-11-25T14:23:11,893 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=581 is superior to 500
2024-11-25T14:23:11,904 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=581, OpenFileDescriptor=925, MaxFileDescriptor=1048576, SystemLoadAverage=439, ProcessCount=11, AvailableMemoryMB=4092
2024-11-25T14:23:11,904 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=581 is superior to 500
2024-11-25T14:23:11,905 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:23:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=236, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:23:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=236
2024-11-25T14:23:11,907 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:11,908 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0e1c5258 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6fe7b85b
2024-11-25T14:23:11,950 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x0e1c5258 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0e1c5258 to 127.0.0.1:51135
2024-11-25T14:23:11,950 WARN  [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:11,983 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=237, ppid=236, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=236
2024-11-25T14:23:12,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=237
2024-11-25T14:23:12,136 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=237}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:23:12,169 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=237}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:12,171 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=237}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=237
2024-11-25T14:23:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=237
2024-11-25T14:23:12,174 INFO  [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:12,175 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=237, resume processing ppid=236
2024-11-25T14:23:12,175 INFO  [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:23:12,175 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=237, ppid=236, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec
2024-11-25T14:23:12,177 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=236, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 270 msec
2024-11-25T14:23:12,187 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:12,189 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ff00f6d to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7871183f
2024-11-25T14:23:12,199 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:12,199 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=236
2024-11-25T14:23:12,223 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:12,224 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:12,225 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=238, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=238
2024-11-25T14:23:12,229 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:12,229 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:12,230 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:12,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:12,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:12,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:12,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:12,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:12,245 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=239, ppid=238, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=238
2024-11-25T14:23:12,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=239
2024-11-25T14:23:12,399 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:23:12,399 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:23:12,399 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:23:12,400 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:12,400 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:12,403 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x40348f0b to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@734c1e98
2024-11-25T14:23:12,404 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:12,404 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:23:12,407 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=239
2024-11-25T14:23:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=239
2024-11-25T14:23:12,409 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:12,416 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=239, resume processing ppid=238
2024-11-25T14:23:12,416 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:23:12,417 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=239, ppid=238, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 164 msec
2024-11-25T14:23:12,417 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:23:12,422 INFO  [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:23:12,424 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=238, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 198 msec
2024-11-25T14:23:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=238
2024-11-25T14:23:12,542 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:12,543 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:23:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=240, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=240
2024-11-25T14:23:12,545 WARN  [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:12,546 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=240, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:12,548 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=240, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec
2024-11-25T14:23:12,548 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=240
2024-11-25T14:23:12,653 INFO  [ForkJoinPool.commonPool-worker-1 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:23:12,678 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=587 (was 581)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7ff00f6d
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x40348f0b-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x40348f0b
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7ff00f6d-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x7ff00f6d-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x40348f0b-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
 - Thread LEAK? -, OpenFileDescriptor=933 (was 925) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=460 (was 439) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4046 (was 4092)
2024-11-25T14:23:12,679 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=587 is superior to 500
2024-11-25T14:23:12,698 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=587, OpenFileDescriptor=933, MaxFileDescriptor=1048576, SystemLoadAverage=460, ProcessCount=11, AvailableMemoryMB=4040
2024-11-25T14:23:12,698 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=587 is superior to 500
2024-11-25T14:23:12,699 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:23:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=241, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:23:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=241
2024-11-25T14:23:12,703 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:12,704 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x31081c47 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@216c7373
2024-11-25T14:23:12,782 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x31081c47 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x31081c47 to 127.0.0.1:51135
2024-11-25T14:23:12,783 WARN  [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=241
2024-11-25T14:23:12,848 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=242, ppid=241, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:13,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=242
2024-11-25T14:23:13,001 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=242}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:23:13,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=241
2024-11-25T14:23:13,042 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=242}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:13,048 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=242}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=242
2024-11-25T14:23:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=242
2024-11-25T14:23:13,051 INFO  [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:13,054 INFO  [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:23:13,055 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=242, resume processing ppid=241
2024-11-25T14:23:13,055 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=242, ppid=241, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 203 msec
2024-11-25T14:23:13,057 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=241, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 355 msec
2024-11-25T14:23:13,074 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:13,076 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2a7357f6 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@729566e2
2024-11-25T14:23:13,108 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 1 more
2024-11-25T14:23:13,108 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis
2024-11-25T14:23:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=241
2024-11-25T14:23:13,332 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:13,333 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:13,335 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false
2024-11-25T14:23:13,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=243, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure
2024-11-25T14:23:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=243
2024-11-25T14:23:13,340 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:13,342 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1ee4549f to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@229db9a6
2024-11-25T14:23:13,360 DEBUG [ReadOnlyZKClient-127.0.0.1:51135@0x1ee4549f {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1ee4549f to 127.0.0.1:51135
2024-11-25T14:23:13,360 WARN  [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1
java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-25T14:23:13,383 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=244, ppid=243, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=243
2024-11-25T14:23:13,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=244
2024-11-25T14:23:13,538 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG
2024-11-25T14:23:13,540 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=244
2024-11-25T14:23:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=244
2024-11-25T14:23:13,544 INFO  [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:13,547 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=244, resume processing ppid=243
2024-11-25T14:23:13,547 INFO  [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:51135/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false
2024-11-25T14:23:13,547 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=244, ppid=243, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 161 msec
2024-11-25T14:23:13,550 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=243, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 212 msec
2024-11-25T14:23:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=243
2024-11-25T14:23:13,663 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed
2024-11-25T14:23:13,663 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1
2024-11-25T14:23:13,665 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=245, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=245
2024-11-25T14:23:13,670 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:13,670 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:13,671 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:13,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:13,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:13,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:13,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:13,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:13,683 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=246, ppid=245, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=245
2024-11-25T14:23:13,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=246
2024-11-25T14:23:13,837 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:23:13,837 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:23:13,837 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:23:13,837 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries
2024-11-25T14:23:13,837 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:51135' and parent='/hbase-test1'
2024-11-25T14:23:13,839 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2a8fd272 to 127.0.0.1:51135 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@217fdbd7
2024-11-25T14:23:13,840 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster
java.io.InterruptedIOException: null
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?]
	at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 7 more
2024-11-25T14:23:13,842 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:23:13,846 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=246
2024-11-25T14:23:13,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=246
2024-11-25T14:23:13,861 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:13,871 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=246, resume processing ppid=245
2024-11-25T14:23:13,871 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=246, ppid=245, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 178 msec
2024-11-25T14:23:13,871 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:23:13,871 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:23:13,908 INFO  [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:23:13,924 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=245, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 243 msec
2024-11-25T14:23:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=245
2024-11-25T14:23:13,983 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:13,984 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=247, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=247
2024-11-25T14:23:13,988 WARN  [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:13,990 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=247, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:13,992 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=247, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec
2024-11-25T14:23:13,992 WARN  [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=247
2024-11-25T14:23:14,092 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1
2024-11-25T14:23:14,093 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:23:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=248, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=248
2024-11-25T14:23:14,096 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:14,098 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=248, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:14,099 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=248, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec
2024-11-25T14:23:14,100 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=248
2024-11-25T14:23:14,202 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:23:14,237 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=593 (was 587)
Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2a7357f6-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2a7357f6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2a8fd272
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370)
	app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007f6240be2990.run(Unknown Source)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2a8fd272-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2a7357f6-SendThread(127.0.0.1:51135)
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
	app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)

Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51135@0x2a8fd272-EventThread
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
 - Thread LEAK? -, OpenFileDescriptor=941 (was 933) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=460 (was 460), ProcessCount=11 (was 11), AvailableMemoryMB=3938 (was 4040)
2024-11-25T14:23:14,237 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=593 is superior to 500
2024-11-25T14:23:14,258 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=593, OpenFileDescriptor=941, MaxFileDescriptor=1048576, SystemLoadAverage=460, ProcessCount=11, AvailableMemoryMB=3934
2024-11-25T14:23:14,258 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=593 is superior to 500
2024-11-25T14:23:14,260 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:23:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=249, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:23:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=249
2024-11-25T14:23:14,292 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=250, ppid=249, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=249
2024-11-25T14:23:14,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=250
2024-11-25T14:23:14,446 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=250}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER
2024-11-25T14:23:14,489 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=250}] regionserver.ReplicationSource(231): queueId=1-cb69081f5976,36701,1732544544686, ReplicationSource: 1, currentBandwidth=0
2024-11-25T14:23:14,492 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=250}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=250
2024-11-25T14:23:14,493 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(613): peerId=1, queueId=1-cb69081f5976,36701,1732544544686 (queues=1) is replicating from cluster=b9c5d22a-5299-4650-a1d7-40bf093ed607 to cluster=b9c5d22a-5299-4650-a1d7-40bf093ed607
2024-11-25T14:23:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=250
2024-11-25T14:23:14,493 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSource(382): peerId=1, starting shipping worker for walGroupId=cb69081f5976%2C36701%2C1732544544686
2024-11-25T14:23:14,494 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=1-cb69081f5976,36701,1732544544686, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1
2024-11-25T14:23:14,496 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686.replicationSource.wal-reader.cb69081f5976%2C36701%2C1732544544686,1-cb69081f5976,36701,1732544544686 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/WALs/cb69081f5976,36701,1732544544686/cb69081f5976%2C36701%2C1732544544686.1732544547907, startPosition=0, beingWritten=true
2024-11-25T14:23:14,497 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:14,498 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686.replicationSource.shippercb69081f5976%2C36701%2C1732544544686,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: cb69081f5976%2C36701%2C1732544544686
2024-11-25T14:23:14,500 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=250, resume processing ppid=249
2024-11-25T14:23:14,500 INFO  [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false
2024-11-25T14:23:14,500 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=250, ppid=249, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 205 msec
2024-11-25T14:23:14,502 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=249, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 240 msec
2024-11-25T14:23:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=249
2024-11-25T14:23:14,582 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:14,583 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED
2024-11-25T14:23:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=251, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure
2024-11-25T14:23:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=251
2024-11-25T14:23:14,585 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: 
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: java.io.IOException: Cluster key passed  is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?]
	... 12 more
2024-11-25T14:23:14,586 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=251, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: 
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: 
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
Caused by: java.io.IOException: Cluster key passed  is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:14,587 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=251, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=4 msec
2024-11-25T14:23:14,587 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=251
2024-11-25T14:23:14,693 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 
2024-11-25T14:23:14,693 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1
2024-11-25T14:23:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=252, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252
2024-11-25T14:23:14,697 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:14,697 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:14,698 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:14,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state
2024-11-25T14:23:14,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:14,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state
2024-11-25T14:23:14,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state
2024-11-25T14:23:14,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1
2024-11-25T14:23:14,708 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=253, ppid=252, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}]
2024-11-25T14:23:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252
2024-11-25T14:23:14,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36701 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=253
2024-11-25T14:23:14,860 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER
2024-11-25T14:23:14,861 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0
2024-11-25T14:23:14,861 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-cb69081f5976,36701,1732544544686 because: Replication stream was removed by a user
2024-11-25T14:23:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252
2024-11-25T14:23:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252
2024-11-25T14:23:15,382 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-25T14:23:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252
2024-11-25T14:23:15,861 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686.replicationSource.wal-reader.cb69081f5976%2C36701%2C1732544544686,1-cb69081f5976,36701,1732544544686 {}] util.Threads(127): sleep interrupted
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) ~[classes/:?]
2024-11-25T14:23:15,861 WARN  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686.replicationSource.shippercb69081f5976%2C36701%2C1732544544686,1-cb69081f5976,36701,1732544544686 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch
java.lang.InterruptedException: null
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?]
	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:309) ~[classes/:?]
	at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?]
2024-11-25T14:23:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252
2024-11-25T14:23:16,861 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSource(739): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/cb69081f5976:0-0.replicationSource,1-cb69081f5976,36701,1732544544686.replicationSource.shippercb69081f5976%2C36701%2C1732544544686,1-cb69081f5976,36701,1732544544686 terminated
2024-11-25T14:23:16,861 INFO  [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSourceManager(562): Done with the queue 1-cb69081f5976,36701,1732544544686
2024-11-25T14:23:16,864 DEBUG [RS_REFRESH_PEER-regionserver/cb69081f5976:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=253
2024-11-25T14:23:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(4169): Remote procedure done, pid=253
2024-11-25T14:23:16,866 INFO  [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on cb69081f5976,36701,1732544544686 suceeded
2024-11-25T14:23:16,869 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=253, resume processing ppid=252
2024-11-25T14:23:16,869 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on
2024-11-25T14:23:16,869 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on
2024-11-25T14:23:16,869 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=253, ppid=252, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1580 sec
2024-11-25T14:23:16,876 INFO  [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1
2024-11-25T14:23:16,878 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=252, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.1820 sec
2024-11-25T14:23:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252
2024-11-25T14:23:18,862 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed
2024-11-25T14:23:18,863 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2
2024-11-25T14:23:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] procedure2.ProcedureExecutor(1139): Stored pid=254, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure
2024-11-25T14:23:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=254
2024-11-25T14:23:18,865 WARN  [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up
org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
2024-11-25T14:23:18,865 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=254, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug?
org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?]
	at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?]
	... 8 more
2024-11-25T14:23:18,866 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=254, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec
2024-11-25T14:23:18,866 WARN  [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store.
2024-11-25T14:23:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44721 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=254
2024-11-25T14:23:18,973 INFO  [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2
2024-11-25T14:23:18,993 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=595 (was 593) - Thread LEAK? -, OpenFileDescriptor=946 (was 941) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=455 (was 460), ProcessCount=11 (was 11), AvailableMemoryMB=4490 (was 3934) - AvailableMemoryMB LEAK? -
2024-11-25T14:23:18,993 WARN  [Time-limited test {}] hbase.ResourceChecker(130): Thread=595 is superior to 500
2024-11-25T14:23:18,993 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-25T14:23:18,994 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:96)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-25T14:23:18,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T14:23:18,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T14:23:18,995 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-25T14:23:18,995 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-25T14:23:18,995 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-25T14:23:18,995 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:97)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-25T14:23:18,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T14:23:18,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T14:23:18,995 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-25T14:23:18,996 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-25T14:23:18,996 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1138968186, stopped=false
2024-11-25T14:23:18,996 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=cb69081f5976,44721,1732544543804
2024-11-25T14:23:19,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-25T14:23:19,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-25T14:23:19,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:23:19,007 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-25T14:23:19,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:23:19,007 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-25T14:23:19,007 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:97)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-25T14:23:19,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T14:23:19,007 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-25T14:23:19,008 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cb69081f5976,36701,1732544544686' *****
2024-11-25T14:23:19,008 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-25T14:23:19,008 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-25T14:23:19,008 INFO  [RS:0;cb69081f5976:36701 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-25T14:23:19,009 INFO  [RS:0;cb69081f5976:36701 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-25T14:23:19,009 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(3091): Received CLOSE for c4a88aaabbd3b8251330812a470ab6d5
2024-11-25T14:23:19,009 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-25T14:23:19,010 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-25T14:23:19,011 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(959): stopping server cb69081f5976,36701,1732544544686
2024-11-25T14:23:19,011 INFO  [RS:0;cb69081f5976:36701 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-25T14:23:19,011 INFO  [RS:0;cb69081f5976:36701 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;cb69081f5976:36701.
2024-11-25T14:23:19,011 DEBUG [RS:0;cb69081f5976:36701 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-25T14:23:19,011 DEBUG [RS:0;cb69081f5976:36701 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T14:23:19,011 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-25T14:23:19,011 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-25T14:23:19,011 DEBUG [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c4a88aaabbd3b8251330812a470ab6d5, disabling compactions & flushes
2024-11-25T14:23:19,011 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-25T14:23:19,011 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-25T14:23:19,011 INFO  [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.
2024-11-25T14:23:19,011 DEBUG [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.
2024-11-25T14:23:19,012 DEBUG [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5. after waiting 0 ms
2024-11-25T14:23:19,012 DEBUG [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.
2024-11-25T14:23:19,013 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-25T14:23:19,013 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1325): Online Regions={c4a88aaabbd3b8251330812a470ab6d5=hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5., 1588230740=hbase:meta,,1.1588230740}
2024-11-25T14:23:19,013 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c4a88aaabbd3b8251330812a470ab6d5
2024-11-25T14:23:19,015 DEBUG [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-25T14:23:19,015 INFO  [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-25T14:23:19,016 DEBUG [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-25T14:23:19,016 DEBUG [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-25T14:23:19,016 DEBUG [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-25T14:23:19,017 INFO  [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c4a88aaabbd3b8251330812a470ab6d5 3/3 column families, dataSize=5.05 KB heapSize=10.72 KB
2024-11-25T14:23:19,020 INFO  [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.34 KB heapSize=3.38 KB
2024-11-25T14:23:19,032 INFO  [regionserver/cb69081f5976:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-25T14:23:19,108 DEBUG [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/replication/c4a88aaabbd3b8251330812a470ab6d5/.tmp/hfileref/eb452cc609aa401997cf4eba3a01b027 is 33, key is 1/hfileref:/1732544596875/DeleteFamily/seqid=0
2024-11-25T14:23:19,108 DEBUG [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/.tmp/info/9cbd675e3c2b4bd9b2e493267f483e86 is 147, key is hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5./info:regioninfo/1732544550740/Put/seqid=0
2024-11-25T14:23:19,119 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741839_1015 (size=6631)
2024-11-25T14:23:19,121 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741838_1014 (size=5733)
2024-11-25T14:23:19,121 INFO  [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=638 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/replication/c4a88aaabbd3b8251330812a470ab6d5/.tmp/hfileref/eb452cc609aa401997cf4eba3a01b027
2024-11-25T14:23:19,157 INFO  [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eb452cc609aa401997cf4eba3a01b027
2024-11-25T14:23:19,186 DEBUG [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/replication/c4a88aaabbd3b8251330812a470ab6d5/.tmp/queue/dca3ff9fbee0412492a856c05e5c154e is 63, key is 1-cb69081f5976,36701,1732544544686/queue:/1732544596862/DeleteFamily/seqid=0
2024-11-25T14:23:19,192 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741840_1016 (size=6582)
2024-11-25T14:23:19,213 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c4a88aaabbd3b8251330812a470ab6d5
2024-11-25T14:23:19,414 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c4a88aaabbd3b8251330812a470ab6d5
2024-11-25T14:23:19,520 INFO  [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.17 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/.tmp/info/9cbd675e3c2b4bd9b2e493267f483e86
2024-11-25T14:23:19,550 DEBUG [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/.tmp/ns/22ca3f771e14459491ac5b7811dd04b7 is 43, key is default/ns:d/1732544549116/Put/seqid=0
2024-11-25T14:23:19,555 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741841_1017 (size=5153)
2024-11-25T14:23:19,594 INFO  [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.43 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/replication/c4a88aaabbd3b8251330812a470ab6d5/.tmp/queue/dca3ff9fbee0412492a856c05e5c154e
2024-11-25T14:23:19,604 INFO  [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dca3ff9fbee0412492a856c05e5c154e
2024-11-25T14:23:19,606 DEBUG [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/replication/c4a88aaabbd3b8251330812a470ab6d5/.tmp/hfileref/eb452cc609aa401997cf4eba3a01b027 as hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/replication/c4a88aaabbd3b8251330812a470ab6d5/hfileref/eb452cc609aa401997cf4eba3a01b027
2024-11-25T14:23:19,614 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c4a88aaabbd3b8251330812a470ab6d5
2024-11-25T14:23:19,617 INFO  [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eb452cc609aa401997cf4eba3a01b027
2024-11-25T14:23:19,618 INFO  [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/replication/c4a88aaabbd3b8251330812a470ab6d5/hfileref/eb452cc609aa401997cf4eba3a01b027, entries=22, sequenceid=70, filesize=5.6 K
2024-11-25T14:23:19,622 DEBUG [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/replication/c4a88aaabbd3b8251330812a470ab6d5/.tmp/queue/dca3ff9fbee0412492a856c05e5c154e as hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/replication/c4a88aaabbd3b8251330812a470ab6d5/queue/dca3ff9fbee0412492a856c05e5c154e
2024-11-25T14:23:19,637 INFO  [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dca3ff9fbee0412492a856c05e5c154e
2024-11-25T14:23:19,637 INFO  [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/replication/c4a88aaabbd3b8251330812a470ab6d5/queue/dca3ff9fbee0412492a856c05e5c154e, entries=22, sequenceid=70, filesize=6.4 K
2024-11-25T14:23:19,644 INFO  [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~5.05 KB/5170, heapSize ~10.44 KB/10688, currentSize=0 B/0 for c4a88aaabbd3b8251330812a470ab6d5 in 627ms, sequenceid=70, compaction requested=false
2024-11-25T14:23:19,653 DEBUG [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/replication/c4a88aaabbd3b8251330812a470ab6d5/recovered.edits/73.seqid, newMaxSeqId=73, maxSeqId=1
2024-11-25T14:23:19,656 DEBUG [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-25T14:23:19,656 INFO  [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.
2024-11-25T14:23:19,656 DEBUG [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c4a88aaabbd3b8251330812a470ab6d5:
Waiting for close lock at 1732544599011Running coprocessor pre-close hooks at 1732544599011Disabling compacts and flushes for region at 1732544599011Disabling writes for close at 1732544599012 (+1 ms)Obtaining lock to block concurrent updates at 1732544599017 (+5 ms)Preparing flush snapshotting stores in c4a88aaabbd3b8251330812a470ab6d5 at 1732544599017Finished memstore snapshotting hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5., syncing WAL and waiting on mvcc, flushsize=dataSize=5170, getHeapSize=10928, getOffHeapSize=0, getCellsCount=66 at 1732544599026 (+9 ms)Flushing stores of hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5. at 1732544599033 (+7 ms)Flushing c4a88aaabbd3b8251330812a470ab6d5/hfileref: creating writer at 1732544599033Flushing c4a88aaabbd3b8251330812a470ab6d5/hfileref: appending metadata at 1732544599093 (+60 ms)Flushing c4a88aaabbd3b8251330812a470ab6d5/hfileref: closing flushed file at 1732544599095 (+2 ms)Flushing c4a88aaabbd3b8251330812a470ab6d5/queue: creating writer at 1732544599170 (+75 ms)Flushing c4a88aaabbd3b8251330812a470ab6d5/queue: appending metadata at 1732544599184 (+14 ms)Flushing c4a88aaabbd3b8251330812a470ab6d5/queue: closing flushed file at 1732544599184Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@341c9a5b: reopening flushed file at 1732544599604 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f437ec8: reopening flushed file at 1732544599620 (+16 ms)Finished flush of dataSize ~5.05 KB/5170, heapSize ~10.44 KB/10688, currentSize=0 B/0 for c4a88aaabbd3b8251330812a470ab6d5 in 627ms, sequenceid=70, compaction requested=false at 1732544599644 (+24 ms)Writing region close event to WAL at 1732544599646 (+2 ms)Running coprocessor post-close hooks at 1732544599654 (+8 ms)Closed at 1732544599656 (+2 ms)
2024-11-25T14:23:19,657 DEBUG [RS_CLOSE_REGION-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:replication,,1732544549441.c4a88aaabbd3b8251330812a470ab6d5.
2024-11-25T14:23:19,743 INFO  [regionserver/cb69081f5976:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-25T14:23:19,743 INFO  [regionserver/cb69081f5976:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-25T14:23:19,814 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-25T14:23:19,960 INFO  [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/.tmp/ns/22ca3f771e14459491ac5b7811dd04b7
2024-11-25T14:23:19,990 DEBUG [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/.tmp/table/719537bce1434616b105a100a1b1d75e is 53, key is hbase:replication/table:state/1732544550771/Put/seqid=0
2024-11-25T14:23:19,999 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741842_1018 (size=5256)
2024-11-25T14:23:20,014 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-25T14:23:20,014 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-25T14:23:20,015 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-25T14:23:20,215 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-25T14:23:20,401 INFO  [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=98 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/.tmp/table/719537bce1434616b105a100a1b1d75e
2024-11-25T14:23:20,414 DEBUG [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/.tmp/info/9cbd675e3c2b4bd9b2e493267f483e86 as hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/info/9cbd675e3c2b4bd9b2e493267f483e86
2024-11-25T14:23:20,415 DEBUG [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-25T14:23:20,422 INFO  [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/info/9cbd675e3c2b4bd9b2e493267f483e86, entries=10, sequenceid=11, filesize=6.5 K
2024-11-25T14:23:20,423 DEBUG [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/.tmp/ns/22ca3f771e14459491ac5b7811dd04b7 as hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/ns/22ca3f771e14459491ac5b7811dd04b7
2024-11-25T14:23:20,431 INFO  [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/ns/22ca3f771e14459491ac5b7811dd04b7, entries=2, sequenceid=11, filesize=5.0 K
2024-11-25T14:23:20,433 DEBUG [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/.tmp/table/719537bce1434616b105a100a1b1d75e as hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/table/719537bce1434616b105a100a1b1d75e
2024-11-25T14:23:20,442 INFO  [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/table/719537bce1434616b105a100a1b1d75e, entries=2, sequenceid=11, filesize=5.1 K
2024-11-25T14:23:20,443 INFO  [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.34 KB/1368, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 1427ms, sequenceid=11, compaction requested=false
2024-11-25T14:23:20,449 DEBUG [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-25T14:23:20,450 DEBUG [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-25T14:23:20,450 INFO  [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-25T14:23:20,450 DEBUG [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732544599015Running coprocessor pre-close hooks at 1732544599015Disabling compacts and flushes for region at 1732544599015Disabling writes for close at 1732544599016 (+1 ms)Obtaining lock to block concurrent updates at 1732544599020 (+4 ms)Preparing flush snapshotting stores in 1588230740 at 1732544599020Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1368, getHeapSize=3392, getOffHeapSize=0, getCellsCount=14 at 1732544599026 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732544599028 (+2 ms)Flushing 1588230740/info: creating writer at 1732544599033 (+5 ms)Flushing 1588230740/info: appending metadata at 1732544599093 (+60 ms)Flushing 1588230740/info: closing flushed file at 1732544599095 (+2 ms)Flushing 1588230740/ns: creating writer at 1732544599531 (+436 ms)Flushing 1588230740/ns: appending metadata at 1732544599549 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732544599549Flushing 1588230740/table: creating writer at 1732544599972 (+423 ms)Flushing 1588230740/table: appending metadata at 1732544599990 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732544599990Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@260de28e: reopening flushed file at 1732544600413 (+423 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b2d0141: reopening flushed file at 1732544600422 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35dde814: reopening flushed file at 1732544600432 (+10 ms)Finished flush of dataSize ~1.34 KB/1368, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 1427ms, sequenceid=11, compaction requested=false at 1732544600443 (+11 ms)Writing region close event to WAL at 1732544600444 (+1 ms)Running coprocessor post-close hooks at 1732544600449 (+5 ms)Closed at 1732544600450 (+1 ms)
2024-11-25T14:23:20,450 DEBUG [RS_CLOSE_META-regionserver/cb69081f5976:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-25T14:23:20,615 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(976): stopping server cb69081f5976,36701,1732544544686; all regions closed.
2024-11-25T14:23:20,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741834_1010 (size=2742)
2024-11-25T14:23:20,631 DEBUG [RS:0;cb69081f5976:36701 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/oldWALs
2024-11-25T14:23:20,631 INFO  [RS:0;cb69081f5976:36701 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cb69081f5976%2C36701%2C1732544544686.meta:.meta(num 1732544548922)
2024-11-25T14:23:20,633 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741837_1013 (size=11128)
2024-11-25T14:23:20,634 WARN  [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/WALs/cb69081f5976,36701,1732544544686/cb69081f5976%2C36701%2C1732544544686.rep.1732544550656 not finished, retry = 0
2024-11-25T14:23:20,737 DEBUG [RS:0;cb69081f5976:36701 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/oldWALs
2024-11-25T14:23:20,737 INFO  [RS:0;cb69081f5976:36701 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cb69081f5976%2C36701%2C1732544544686.rep:(num 1732544550656)
2024-11-25T14:23:20,740 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741832_1008 (size=93)
2024-11-25T14:23:20,740 WARN  [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/WALs/cb69081f5976,36701,1732544544686/cb69081f5976%2C36701%2C1732544544686.1732544547907 not finished, retry = 0
2024-11-25T14:23:20,844 DEBUG [RS:0;cb69081f5976:36701 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/oldWALs
2024-11-25T14:23:20,844 INFO  [RS:0;cb69081f5976:36701 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cb69081f5976%2C36701%2C1732544544686:(num 1732544547907)
2024-11-25T14:23:20,844 DEBUG [RS:0;cb69081f5976:36701 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T14:23:20,844 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.LeaseManager(133): Closed leases
2024-11-25T14:23:20,844 INFO  [RS:0;cb69081f5976:36701 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-25T14:23:20,845 INFO  [RS:0;cb69081f5976:36701 {}] hbase.ChoreService(370): Chore service for: regionserver/cb69081f5976:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-25T14:23:20,845 INFO  [RS:0;cb69081f5976:36701 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-25T14:23:20,845 INFO  [regionserver/cb69081f5976:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-25T14:23:20,846 INFO  [RS:0;cb69081f5976:36701 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36701
2024-11-25T14:23:20,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-25T14:23:20,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cb69081f5976,36701,1732544544686
2024-11-25T14:23:20,857 INFO  [RS:0;cb69081f5976:36701 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-25T14:23:20,865 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cb69081f5976,36701,1732544544686]
2024-11-25T14:23:20,873 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cb69081f5976,36701,1732544544686 already deleted, retry=false
2024-11-25T14:23:20,873 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; cb69081f5976,36701,1732544544686 expired; onlineServers=0
2024-11-25T14:23:20,873 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'cb69081f5976,44721,1732544543804' *****
2024-11-25T14:23:20,874 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-25T14:23:20,874 INFO  [M:0;cb69081f5976:44721 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-25T14:23:20,874 INFO  [M:0;cb69081f5976:44721 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-25T14:23:20,874 DEBUG [M:0;cb69081f5976:44721 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-25T14:23:20,874 DEBUG [M:0;cb69081f5976:44721 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-25T14:23:20,874 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-25T14:23:20,874 DEBUG [master/cb69081f5976:0:becomeActiveMaster-HFileCleaner.small.0-1732544547622 {}] cleaner.HFileCleaner(306): Exit Thread[master/cb69081f5976:0:becomeActiveMaster-HFileCleaner.small.0-1732544547622,5,FailOnTimeoutGroup]
2024-11-25T14:23:20,874 DEBUG [master/cb69081f5976:0:becomeActiveMaster-HFileCleaner.large.0-1732544547621 {}] cleaner.HFileCleaner(306): Exit Thread[master/cb69081f5976:0:becomeActiveMaster-HFileCleaner.large.0-1732544547621,5,FailOnTimeoutGroup]
2024-11-25T14:23:20,874 INFO  [M:0;cb69081f5976:44721 {}] hbase.ChoreService(370): Chore service for: master/cb69081f5976:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-25T14:23:20,875 INFO  [M:0;cb69081f5976:44721 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-25T14:23:20,875 DEBUG [M:0;cb69081f5976:44721 {}] master.HMaster(1795): Stopping service threads
2024-11-25T14:23:20,875 INFO  [M:0;cb69081f5976:44721 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-25T14:23:20,875 INFO  [M:0;cb69081f5976:44721 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-25T14:23:20,875 INFO  [M:0;cb69081f5976:44721 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-25T14:23:20,875 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-25T14:23:20,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-25T14:23:20,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-25T14:23:20,881 DEBUG [M:0;cb69081f5976:44721 {}] zookeeper.ZKUtil(347): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-25T14:23:20,882 WARN  [M:0;cb69081f5976:44721 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-25T14:23:20,883 INFO  [M:0;cb69081f5976:44721 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/.lastflushedseqids
2024-11-25T14:23:20,894 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741843_1019 (size=172)
2024-11-25T14:23:20,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-25T14:23:20,965 INFO  [RS:0;cb69081f5976:36701 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-25T14:23:20,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36701-0x10173a96ace0001, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-25T14:23:20,966 INFO  [RS:0;cb69081f5976:36701 {}] regionserver.HRegionServer(1031): Exiting; stopping=cb69081f5976,36701,1732544544686; zookeeper connection closed.
2024-11-25T14:23:20,966 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@14cab15d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@14cab15d
2024-11-25T14:23:20,967 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-25T14:23:21,295 INFO  [M:0;cb69081f5976:44721 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-25T14:23:21,295 INFO  [M:0;cb69081f5976:44721 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-25T14:23:21,295 DEBUG [M:0;cb69081f5976:44721 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-25T14:23:21,295 INFO  [M:0;cb69081f5976:44721 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-25T14:23:21,296 DEBUG [M:0;cb69081f5976:44721 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-25T14:23:21,296 DEBUG [M:0;cb69081f5976:44721 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-25T14:23:21,296 DEBUG [M:0;cb69081f5976:44721 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-25T14:23:21,296 INFO  [M:0;cb69081f5976:44721 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=748.80 KB heapSize=894.09 KB
2024-11-25T14:23:21,312 DEBUG [M:0;cb69081f5976:44721 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3e30d64bf55b4dc9ab1fc3fb67fcb201 is 82, key is hbase:meta,,1/info:regioninfo/1732544549014/Put/seqid=0
2024-11-25T14:23:21,317 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741844_1020 (size=5672)
2024-11-25T14:23:21,718 INFO  [M:0;cb69081f5976:44721 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1763 (bloomFilter=true), to=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3e30d64bf55b4dc9ab1fc3fb67fcb201
2024-11-25T14:23:21,754 DEBUG [M:0;cb69081f5976:44721 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/65696da9bdb84f118dac46d7eaeafeb1 is 2004, key is \x00\x00\x00\x00\x00\x00\x00\x9B/proc:d/1732544579311/Put/seqid=0
2024-11-25T14:23:21,756 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741845_1021 (size=162091)
2024-11-25T14:23:21,922 WARN  [Async-Client-Retry-Timer-pool-0 {}] client.RawAsyncHBaseAdmin(3035): failed to get the procedure result procId=22
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=cb69081f5976:44721 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub.getProcedureResult(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin.lambda$getProcedureResult$257(RawAsyncHBaseAdmin.java:3031) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin.call(RawAsyncHBaseAdmin.java:460) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin.lambda$getProcedureResult$259(RawAsyncHBaseAdmin.java:3029) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$5(AsyncMasterRequestRpcRetryingCaller.java:74) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?]
	at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.doCall(AsyncMasterRequestRpcRetryingCaller.java:67) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$MasterRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:475) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin.getProcedureResult(RawAsyncHBaseAdmin.java:3032) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin.lambda$getProcedureResult$261(RawAsyncHBaseAdmin.java:3042) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 22 more
2024-11-25T14:23:22,157 INFO  [M:0;cb69081f5976:44721 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746.25 KB at sequenceid=1763 (bloomFilter=true), to=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/65696da9bdb84f118dac46d7eaeafeb1
2024-11-25T14:23:22,167 INFO  [M:0;cb69081f5976:44721 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 65696da9bdb84f118dac46d7eaeafeb1
2024-11-25T14:23:22,188 DEBUG [M:0;cb69081f5976:44721 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e7008552eaa44fd69ca09b63498e2e8a is 69, key is cb69081f5976,36701,1732544544686/rs:state/1732544547650/Put/seqid=0
2024-11-25T14:23:22,194 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741846_1022 (size=5156)
2024-11-25T14:23:22,363 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-25T14:23:22,595 INFO  [M:0;cb69081f5976:44721 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1763 (bloomFilter=true), to=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e7008552eaa44fd69ca09b63498e2e8a
2024-11-25T14:23:22,625 DEBUG [M:0;cb69081f5976:44721 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/70799086a7da42d786ba383089b89f16 is 68, key is replication_peer_modification_on/state:d/1732544598976/Put/seqid=0
2024-11-25T14:23:22,633 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741847_1023 (size=5154)
2024-11-25T14:23:22,634 INFO  [M:0;cb69081f5976:44721 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2 KB at sequenceid=1763 (bloomFilter=true), to=hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/70799086a7da42d786ba383089b89f16
2024-11-25T14:23:22,646 DEBUG [M:0;cb69081f5976:44721 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3e30d64bf55b4dc9ab1fc3fb67fcb201 as hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3e30d64bf55b4dc9ab1fc3fb67fcb201
2024-11-25T14:23:22,657 INFO  [M:0;cb69081f5976:44721 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3e30d64bf55b4dc9ab1fc3fb67fcb201, entries=8, sequenceid=1763, filesize=5.5 K
2024-11-25T14:23:22,660 DEBUG [M:0;cb69081f5976:44721 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/65696da9bdb84f118dac46d7eaeafeb1 as hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/65696da9bdb84f118dac46d7eaeafeb1
2024-11-25T14:23:22,671 INFO  [M:0;cb69081f5976:44721 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 65696da9bdb84f118dac46d7eaeafeb1
2024-11-25T14:23:22,671 INFO  [M:0;cb69081f5976:44721 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/65696da9bdb84f118dac46d7eaeafeb1, entries=254, sequenceid=1763, filesize=158.3 K
2024-11-25T14:23:22,673 DEBUG [M:0;cb69081f5976:44721 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e7008552eaa44fd69ca09b63498e2e8a as hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e7008552eaa44fd69ca09b63498e2e8a
2024-11-25T14:23:22,683 INFO  [M:0;cb69081f5976:44721 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e7008552eaa44fd69ca09b63498e2e8a, entries=1, sequenceid=1763, filesize=5.0 K
2024-11-25T14:23:22,685 DEBUG [M:0;cb69081f5976:44721 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/70799086a7da42d786ba383089b89f16 as hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/70799086a7da42d786ba383089b89f16
2024-11-25T14:23:22,694 INFO  [M:0;cb69081f5976:44721 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38307/user/jenkins/test-data/ca498eb8-42f1-de2c-02d0-f9ca7cf53581/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/70799086a7da42d786ba383089b89f16, entries=1, sequenceid=1763, filesize=5.0 K
2024-11-25T14:23:22,696 INFO  [M:0;cb69081f5976:44721 {}] regionserver.HRegion(3140): Finished flush of dataSize ~748.80 KB/766774, heapSize ~894.02 KB/915480, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1399ms, sequenceid=1763, compaction requested=false
2024-11-25T14:23:22,704 INFO  [M:0;cb69081f5976:44721 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-25T14:23:22,705 DEBUG [M:0;cb69081f5976:44721 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732544601295Disabling compacts and flushes for region at 1732544601295Disabling writes for close at 1732544601296 (+1 ms)Obtaining lock to block concurrent updates at 1732544601296Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732544601296Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=766774, getHeapSize=915480, getOffHeapSize=0, getCellsCount=1954 at 1732544601296Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732544601297 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732544601297Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732544601312 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732544601312Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732544601726 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732544601754 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732544601754Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732544602167 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732544602188 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732544602188Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732544602604 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732544602624 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732544602624Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@162ca7bc: reopening flushed file at 1732544602644 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a268744: reopening flushed file at 1732544602657 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50980aa9: reopening flushed file at 1732544602671 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43ae3847: reopening flushed file at 1732544602683 (+12 ms)Finished flush of dataSize ~748.80 KB/766774, heapSize ~894.02 KB/915480, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1399ms, sequenceid=1763, compaction requested=false at 1732544602696 (+13 ms)Writing region close event to WAL at 1732544602704 (+8 ms)Closed at 1732544602704
2024-11-25T14:23:22,710 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37535 is added to blk_1073741830_1006 (size=882586)
2024-11-25T14:23:22,711 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-25T14:23:22,711 INFO  [M:0;cb69081f5976:44721 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-25T14:23:22,711 INFO  [M:0;cb69081f5976:44721 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44721
2024-11-25T14:23:22,712 INFO  [M:0;cb69081f5976:44721 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-25T14:23:22,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-25T14:23:22,832 INFO  [M:0;cb69081f5976:44721 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-25T14:23:22,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44721-0x10173a96ace0000, quorum=127.0.0.1:51135, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-25T14:23:22,863 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2e64f7c7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-25T14:23:22,865 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@577c436c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-25T14:23:22,865 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-25T14:23:22,865 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4043ca1d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-25T14:23:22,865 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5049cee1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/hadoop.log.dir/,STOPPED}
2024-11-25T14:23:22,867 WARN  [BP-610877534-172.17.0.3-1732544540075 heartbeating to localhost/127.0.0.1:38307 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-610877534-172.17.0.3-1732544540075 (Datanode Uuid 71803459-73b8-4a49-97a8-eed8a0826365) service to localhost/127.0.0.1:38307
2024-11-25T14:23:22,867 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-25T14:23:22,867 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-25T14:23:22,868 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/cluster_f33e2636-644a-9dd7-bdf8-442b0e715513/data/data1/current/BP-610877534-172.17.0.3-1732544540075 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-25T14:23:22,868 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/cluster_f33e2636-644a-9dd7-bdf8-442b0e715513/data/data2/current/BP-610877534-172.17.0.3-1732544540075 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-25T14:23:22,868 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-25T14:23:22,873 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@38149812{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-25T14:23:22,874 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17efcf28{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-25T14:23:22,874 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-25T14:23:22,874 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7cc143ab{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-25T14:23:22,874 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@500f17d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d1709441-2ce9-0c9d-127a-66afe9fcde91/hadoop.log.dir/,STOPPED}
2024-11-25T14:23:22,890 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-25T14:23:22,908 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down